[llvm] [AMDGPU][GlobalISel] Add RegBankLegalize rules for atomic fmin/fmax (PR #182824)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 17 06:38:36 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/182824
>From 5a5646954b091be2ae6dfedb83570e34c8d1c9e8 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Mon, 23 Feb 2026 02:55:03 -0600
Subject: [PATCH] [AMDGPU][GlobalISel] Add RegBankLegalize rules for atomic
fmin/fmax
Add register bank legalization rules for
G_ATOMICRMW_FMIN/G_ATOMICRMW_FMAX (flat, global, LDS)
and G_AMDGPU_BUFFER_ATOMIC_FMIN/G_AMDGPU_BUFFER_ATOMIC_FMAX
(S32 and S64) under -new-reg-bank-select. Update
existing GlobalISel tests to use the new pass and
add a new MIR test for register bank assignment.
---
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 5 +-
.../AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll | 596 ++++++++++++++++++
.../AMDGPU/GlobalISel/atomicrmw_fmax.ll | 146 +++--
.../AMDGPU/GlobalISel/atomicrmw_fmin.ll | 146 +++--
.../AMDGPU/fp-min-max-buffer-atomics.ll | 12 +-
.../AMDGPU/fp-min-max-buffer-ptr-atomics.ll | 10 +-
.../AMDGPU/fp64-min-max-buffer-atomics.ll | 8 +-
.../AMDGPU/fp64-min-max-buffer-ptr-atomics.ll | 8 +-
8 files changed, 814 insertions(+), 117 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index acb9bfb703e24..e6341f0d682c3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -799,7 +799,7 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN,
G_ATOMICRMW_UMAX, G_ATOMICRMW_UINC_WRAP,
- G_ATOMICRMW_UDEC_WRAP})
+ G_ATOMICRMW_UDEC_WRAP, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
.Any({{DivS32, P0, S32}, {{Vgpr32}, {VgprP0, Vgpr32}}})
.Any({{DivS64, P0, S64}, {{Vgpr64}, {VgprP0, Vgpr64}}})
.Any({{DivS32, P1, S32}, {{Vgpr32}, {VgprP1, Vgpr32}}})
@@ -846,7 +846,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForGOpcs({G_AMDGPU_BUFFER_ATOMIC_SWAP, G_AMDGPU_BUFFER_ATOMIC_UMAX,
G_AMDGPU_BUFFER_ATOMIC_UMIN, G_AMDGPU_BUFFER_ATOMIC_SMAX,
- G_AMDGPU_BUFFER_ATOMIC_SMIN},
+ G_AMDGPU_BUFFER_ATOMIC_SMIN, G_AMDGPU_BUFFER_ATOMIC_FMAX,
+ G_AMDGPU_BUFFER_ATOMIC_FMIN},
Standard)
.Div(S32, {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Div(S64, {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll
new file mode 100644
index 0000000000000..1248f4a85d927
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll
@@ -0,0 +1,596 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -stop-after=instruction-select < %s | FileCheck %s --check-prefixes=GFX10
+
+; --- atomicrmw fmax: global f32/f64, flat f32/f64, local f32/f64 ---
+
+define void @atomicrmw_fmax_global_f32_vv_noret(ptr addrspace(1) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_global_f32_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: GLOBAL_ATOMIC_FMAX [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmax ptr addrspace(1) %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define float @atomicrmw_fmax_global_f32_vv_ret(ptr addrspace(1) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_global_f32_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[GLOBAL_ATOMIC_FMAX_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_FMAX_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_FMAX_RTN]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+ %old = atomicrmw fmax ptr addrspace(1) %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %old
+}
+
+define void @atomicrmw_fmax_global_f64_vv_noret(ptr addrspace(1) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_global_f64_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: GLOBAL_ATOMIC_MAX_F64 [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmax ptr addrspace(1) %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define double @atomicrmw_fmax_global_f64_vv_ret(ptr addrspace(1) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_global_f64_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: [[GLOBAL_ATOMIC_MAX_F64_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_MAX_F64_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 1, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_MAX_F64_RTN]].sub0
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_MAX_F64_RTN]].sub1
+ ; GFX10-NEXT: $vgpr0 = COPY [[COPY4]]
+ ; GFX10-NEXT: $vgpr1 = COPY [[COPY5]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ %old = atomicrmw fmax ptr addrspace(1) %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %old
+}
+
+define void @atomicrmw_fmax_flat_f32_vv_noret(ptr %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_flat_f32_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: FLAT_ATOMIC_FMAX [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst (s32) on %ir.ptr)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmax ptr %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define float @atomicrmw_fmax_flat_f32_vv_ret(ptr %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_flat_f32_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[FLAT_ATOMIC_FMAX_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_FMAX_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec, implicit $flat_scr :: (load store seq_cst (s32) on %ir.ptr)
+ ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_ATOMIC_FMAX_RTN]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+ %old = atomicrmw fmax ptr %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %old
+}
+
+define void @atomicrmw_fmax_flat_f64_vv_noret(ptr %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_flat_f64_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $src_private_base
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub0
+ ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub1
+ ; GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX10-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 [[COPY1]], [[COPY7]], implicit $exec
+ ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_NE_U32_e64_]]
+ ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY8]], %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.4
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.2.Flow:
+ ; GFX10-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32_xm0_xexec = SI_ELSE [[SI_IF]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.3.atomicrmw.private:
+ ; GFX10-NEXT: successors: %bb.5(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+ ; GFX10-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 0
+ ; GFX10-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[S_MOV_B]]
+ ; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
+ ; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
+ ; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MAX_F64_e64_2:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub0
+ ; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub1
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: S_BRANCH %bb.5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.4.atomicrmw.global:
+ ; GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: FLAT_ATOMIC_MAX_F64 [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst (s64) on %ir.ptr, !noalias.addrspace !1)
+ ; GFX10-NEXT: S_BRANCH %bb.2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.5.atomicrmw.phi:
+ ; GFX10-NEXT: SI_END_CF [[SI_ELSE]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmax ptr %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define double @atomicrmw_fmax_flat_f64_vv_ret(ptr %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_flat_f64_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $src_private_base
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub0
+ ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub1
+ ; GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX10-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 [[COPY1]], [[COPY7]], implicit $exec
+ ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_NE_U32_e64_]]
+ ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY8]], %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.4
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.2.Flow:
+ ; GFX10-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[PHI:%[0-9]+]]:vreg_64 = PHI %19, %bb.4, [[DEF]], %bb.1
+ ; GFX10-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32_xm0_xexec = SI_ELSE [[SI_IF]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.3.atomicrmw.private:
+ ; GFX10-NEXT: successors: %bb.5(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+ ; GFX10-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 0
+ ; GFX10-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[S_MOV_B]]
+ ; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
+ ; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
+ ; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MAX_F64_e64_2:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub0
+ ; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub1
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: S_BRANCH %bb.5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.4.atomicrmw.global:
+ ; GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[FLAT_ATOMIC_MAX_F64_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_MAX_F64_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 1, implicit $exec, implicit $flat_scr :: (load store seq_cst (s64) on %ir.ptr, !noalias.addrspace !1)
+ ; GFX10-NEXT: S_BRANCH %bb.2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.5.atomicrmw.phi:
+ ; GFX10-NEXT: successors: %bb.6(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[PHI1:%[0-9]+]]:vreg_64 = PHI [[PHI]], %bb.2, [[REG_SEQUENCE2]], %bb.3
+ ; GFX10-NEXT: SI_END_CF [[SI_ELSE]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.6.atomicrmw.end:
+ ; GFX10-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[PHI1]].sub0
+ ; GFX10-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[PHI1]].sub1
+ ; GFX10-NEXT: $vgpr0 = COPY [[COPY13]]
+ ; GFX10-NEXT: $vgpr1 = COPY [[COPY14]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ %old = atomicrmw fmax ptr %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %old
+}
+
+define void @atomicrmw_fmax_local_f32_vv_noret(ptr addrspace(3) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_local_f32_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: DS_MAX_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmax ptr addrspace(3) %ptr, float %val seq_cst
+ ret void
+}
+
+define float @atomicrmw_fmax_local_f32_vv_ret(ptr addrspace(3) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_local_f32_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+ %old = atomicrmw fmax ptr addrspace(3) %ptr, float %val seq_cst
+ ret float %old
+}
+
+define void @atomicrmw_fmax_local_f64_vv_noret(ptr addrspace(3) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_local_f64_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX10-NEXT: DS_MAX_F64_gfx9 [[COPY]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmax ptr addrspace(3) %ptr, double %val seq_cst
+ ret void
+}
+
+define double @atomicrmw_fmax_local_f64_vv_ret(ptr addrspace(3) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmax_local_f64_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX10-NEXT: [[DS_MAX_RTN_F64_gfx9_:%[0-9]+]]:vreg_64 = DS_MAX_RTN_F64_gfx9 [[COPY]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[DS_MAX_RTN_F64_gfx9_]].sub0
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[DS_MAX_RTN_F64_gfx9_]].sub1
+ ; GFX10-NEXT: $vgpr0 = COPY [[COPY3]]
+ ; GFX10-NEXT: $vgpr1 = COPY [[COPY4]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ %old = atomicrmw fmax ptr addrspace(3) %ptr, double %val seq_cst
+ ret double %old
+}
+
+; --- atomicrmw fmin: global f32/f64, flat f32/f64, local f32/f64 ---
+
+define void @atomicrmw_fmin_global_f32_vv_noret(ptr addrspace(1) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_global_f32_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: GLOBAL_ATOMIC_FMIN [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmin ptr addrspace(1) %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define float @atomicrmw_fmin_global_f32_vv_ret(ptr addrspace(1) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_global_f32_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[GLOBAL_ATOMIC_FMIN_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_FMIN_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_FMIN_RTN]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+ %old = atomicrmw fmin ptr addrspace(1) %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %old
+}
+
+define void @atomicrmw_fmin_global_f64_vv_noret(ptr addrspace(1) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_global_f64_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: GLOBAL_ATOMIC_MIN_F64 [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmin ptr addrspace(1) %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define double @atomicrmw_fmin_global_f64_vv_ret(ptr addrspace(1) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_global_f64_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: [[GLOBAL_ATOMIC_MIN_F64_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_MIN_F64_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 1, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 1)
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_MIN_F64_RTN]].sub0
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_MIN_F64_RTN]].sub1
+ ; GFX10-NEXT: $vgpr0 = COPY [[COPY4]]
+ ; GFX10-NEXT: $vgpr1 = COPY [[COPY5]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ %old = atomicrmw fmin ptr addrspace(1) %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %old
+}
+
+define void @atomicrmw_fmin_flat_f32_vv_noret(ptr %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_flat_f32_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: FLAT_ATOMIC_FMIN [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst (s32) on %ir.ptr)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmin ptr %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define float @atomicrmw_fmin_flat_f32_vv_ret(ptr %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_flat_f32_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[FLAT_ATOMIC_FMIN_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_FMIN_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec, implicit $flat_scr :: (load store seq_cst (s32) on %ir.ptr)
+ ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_ATOMIC_FMIN_RTN]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+ %old = atomicrmw fmin ptr %ptr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %old
+}
+
+define void @atomicrmw_fmin_flat_f64_vv_noret(ptr %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_flat_f64_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $src_private_base
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub0
+ ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub1
+ ; GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX10-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 [[COPY1]], [[COPY7]], implicit $exec
+ ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_NE_U32_e64_]]
+ ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY8]], %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.4
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.2.Flow:
+ ; GFX10-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32_xm0_xexec = SI_ELSE [[SI_IF]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.3.atomicrmw.private:
+ ; GFX10-NEXT: successors: %bb.5(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+ ; GFX10-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 0
+ ; GFX10-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[S_MOV_B]]
+ ; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
+ ; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
+ ; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MIN_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MIN_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub0
+ ; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub1
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: S_BRANCH %bb.5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.4.atomicrmw.global:
+ ; GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: FLAT_ATOMIC_MIN_F64 [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst (s64) on %ir.ptr, !noalias.addrspace !1)
+ ; GFX10-NEXT: S_BRANCH %bb.2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.5.atomicrmw.phi:
+ ; GFX10-NEXT: SI_END_CF [[SI_ELSE]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmin ptr %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define double @atomicrmw_fmin_flat_f64_vv_ret(ptr %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_flat_f64_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $src_private_base
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub0
+ ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub1
+ ; GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX10-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 [[COPY1]], [[COPY7]], implicit $exec
+ ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_NE_U32_e64_]]
+ ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY8]], %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.4
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.2.Flow:
+ ; GFX10-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[PHI:%[0-9]+]]:vreg_64 = PHI %19, %bb.4, [[DEF]], %bb.1
+ ; GFX10-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32_xm0_xexec = SI_ELSE [[SI_IF]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: S_BRANCH %bb.3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.3.atomicrmw.private:
+ ; GFX10-NEXT: successors: %bb.5(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+ ; GFX10-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 0
+ ; GFX10-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[S_MOV_B]]
+ ; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
+ ; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
+ ; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_MIN_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MIN_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub0
+ ; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub1
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: S_BRANCH %bb.5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.4.atomicrmw.global:
+ ; GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[FLAT_ATOMIC_MIN_F64_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_MIN_F64_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE1]], 0, 1, implicit $exec, implicit $flat_scr :: (load store seq_cst (s64) on %ir.ptr, !noalias.addrspace !1)
+ ; GFX10-NEXT: S_BRANCH %bb.2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.5.atomicrmw.phi:
+ ; GFX10-NEXT: successors: %bb.6(0x80000000)
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[PHI1:%[0-9]+]]:vreg_64 = PHI [[PHI]], %bb.2, [[REG_SEQUENCE2]], %bb.3
+ ; GFX10-NEXT: SI_END_CF [[SI_ELSE]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: bb.6.atomicrmw.end:
+ ; GFX10-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[PHI1]].sub0
+ ; GFX10-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[PHI1]].sub1
+ ; GFX10-NEXT: $vgpr0 = COPY [[COPY13]]
+ ; GFX10-NEXT: $vgpr1 = COPY [[COPY14]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ %old = atomicrmw fmin ptr %ptr, double %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %old
+}
+
+define void @atomicrmw_fmin_local_f32_vv_noret(ptr addrspace(3) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_local_f32_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: DS_MIN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmin ptr addrspace(3) %ptr, float %val seq_cst
+ ret void
+}
+
+define float @atomicrmw_fmin_local_f32_vv_ret(ptr addrspace(3) %ptr, float %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_local_f32_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[DS_MIN_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MIN_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: $vgpr0 = COPY [[DS_MIN_RTN_F32_gfx9_]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+ %old = atomicrmw fmin ptr addrspace(3) %ptr, float %val seq_cst
+ ret float %old
+}
+
+define void @atomicrmw_fmin_local_f64_vv_noret(ptr addrspace(3) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_local_f64_vv_noret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX10-NEXT: DS_MIN_F64_gfx9 [[COPY]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: SI_RETURN
+ %old = atomicrmw fmin ptr addrspace(3) %ptr, double %val seq_cst
+ ret void
+}
+
+define double @atomicrmw_fmin_local_f64_vv_ret(ptr addrspace(3) %ptr, double %val) {
+ ; GFX10-LABEL: name: atomicrmw_fmin_local_f64_vv_ret
+ ; GFX10: bb.1 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX10-NEXT: [[DS_MIN_RTN_F64_gfx9_:%[0-9]+]]:vreg_64 = DS_MIN_RTN_F64_gfx9 [[COPY]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store seq_cst (s64) on %ir.ptr, addrspace 3)
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[DS_MIN_RTN_F64_gfx9_]].sub0
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[DS_MIN_RTN_F64_gfx9_]].sub1
+ ; GFX10-NEXT: $vgpr0 = COPY [[COPY3]]
+ ; GFX10-NEXT: $vgpr1 = COPY [[COPY4]]
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ %old = atomicrmw fmin ptr addrspace(3) %ptr, double %val seq_cst
+ ret double %old
+}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
index 9adb56cb0861e..1973dd8cb58ff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
; TODO: Delete this and add run lines to use *-atomicrmw-fmax.ll tests
@@ -1509,13 +1509,15 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_max_f32_e32 v3, v1, v1
+; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_mov_b32_e32 v0, s6
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
; GFX942-NEXT: v_max_f32_e32 v4, v0, v3
@@ -1559,13 +1561,14 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v0, s6
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
; GFX90A-NEXT: v_max_f32_e32 v4, v0, v3
@@ -1585,13 +1588,14 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v3, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, s6
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
; GFX908-NEXT: v_max_f32_e32 v4, v0, v3
@@ -1612,13 +1616,14 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v1
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
; GFX8-NEXT: v_max_f32_e32 v4, v0, v3
@@ -1669,9 +1674,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_mov_b32_e32 v1, s6
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
; GFX942-NEXT: v_max_f32_e32 v0, v0, v3
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
@@ -1718,9 +1726,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
@@ -1743,9 +1753,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
+; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
; GFX908-NEXT: v_max_f32_e32 v0, v0, v3
; GFX908-NEXT: v_mov_b32_e32 v5, v1
@@ -1769,9 +1781,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
; GFX8-NEXT: v_max_f32_e32 v0, v0, v3
; GFX8-NEXT: v_mov_b32_e32 v5, v1
@@ -1810,17 +1824,22 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], null offen
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], null offen
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_mov_b32_e32 v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -1852,17 +1871,21 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], 0 offen
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s4, v2
+; GFX11-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], 0 offen
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen glc
@@ -1903,12 +1926,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v8, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: v_mov_b32_e32 v4, s6
+; GFX908-NEXT: v_mov_b32_e32 v5, s7
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
@@ -1932,12 +1959,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v8, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-NEXT: v_mov_b32_e32 v5, s7
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
@@ -1979,16 +2010,22 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
-; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], null offen
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -2019,16 +2056,21 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], 0 offen
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s4, v2
+; GFX11-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2071,9 +2113,13 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v3, s7
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v10, v3
@@ -2100,9 +2146,13 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v10, v3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
index 876eacb763695..ea5a9b0d22a15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
; TODO: Delete this and add run lines to use *-atomicrmw-fmin.ll tests
@@ -1509,13 +1509,15 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_max_f32_e32 v3, v1, v1
+; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_mov_b32_e32 v0, s6
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
; GFX942-NEXT: v_min_f32_e32 v4, v0, v3
@@ -1559,13 +1561,14 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v0, s6
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
; GFX90A-NEXT: v_min_f32_e32 v4, v0, v3
@@ -1585,13 +1588,14 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v3, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, s6
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
; GFX908-NEXT: v_min_f32_e32 v4, v0, v3
@@ -1612,13 +1616,14 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v1
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
; GFX8-NEXT: v_min_f32_e32 v4, v0, v3
@@ -1669,9 +1674,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_mov_b32_e32 v1, s6
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
; GFX942-NEXT: v_min_f32_e32 v0, v0, v3
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
@@ -1718,9 +1726,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
@@ -1743,9 +1753,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
+; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
; GFX908-NEXT: v_min_f32_e32 v0, v0, v3
; GFX908-NEXT: v_mov_b32_e32 v5, v1
@@ -1769,9 +1781,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
; GFX8-NEXT: v_min_f32_e32 v0, v0, v3
; GFX8-NEXT: v_mov_b32_e32 v5, v1
@@ -1810,17 +1824,22 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], null offen
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], null offen
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_mov_b32_e32 v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -1852,17 +1871,21 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], 0 offen
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s4, v2
+; GFX11-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], 0 offen
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen glc
@@ -1903,12 +1926,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v8, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: v_mov_b32_e32 v4, s6
+; GFX908-NEXT: v_mov_b32_e32 v5, s7
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
@@ -1932,12 +1959,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v8, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-NEXT: v_mov_b32_e32 v5, s7
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
@@ -1979,16 +2010,22 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
-; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], null offen
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -2019,16 +2056,21 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], 0 offen
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s4, v2
+; GFX11-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2071,9 +2113,13 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s6, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v3, s7
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v10, v3
@@ -2100,9 +2146,13 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v10, v3
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
index f3dd4cbeda16d..e309d36f6fb05 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
@@ -6,12 +6,12 @@
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=GFX1100
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck %s -check-prefix=GFX12
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=G_GFX1100
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 | FileCheck %s -check-prefix=GFX12
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=G_GFX1100
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1200 | FileCheck %s -check-prefix=GFX12
declare float @llvm.amdgcn.raw.buffer.atomic.fmin.f32(float, <4 x i32>, i32, i32, i32 immarg)
declare float @llvm.amdgcn.raw.buffer.atomic.fmax.f32(float, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
index fd4046a70687d..50b15260cd3c8 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
@@ -5,11 +5,11 @@
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=GFX1030
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=GFX1100
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=G_GFX1100
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=G_GFX1100
declare float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
declare float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
index 2d38924a28da4..e987190e335d3 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
@@ -4,10 +4,10 @@
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=GFX10
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=GFX1030
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
declare double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double, <4 x i32>, i32, i32, i32 immarg)
declare double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
index b8363da4e4887..2a7027c3393b2 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
@@ -4,10 +4,10 @@
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=GFX10
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=GFX1030
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
-; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=verde | FileCheck %s -check-prefix=G_SI
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=G_GFX7
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s -check-prefix=G_GFX10
+; RUN: llc < %s -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1030 | FileCheck %s -check-prefix=G_GFX1030
declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double, ptr addrspace(8), i32, i32, i32 immarg)
declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double, ptr addrspace(8), i32, i32, i32 immarg)
More information about the llvm-commits
mailing list