[llvm] 8bba070 - AMDGPU: Expand testing of atomicrmw fmin/fmax lowering
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 25 14:43:01 PDT 2024
Author: Matt Arsenault
Date: 2024-06-25T23:42:48+02:00
New Revision: 8bba070ef84ce41be4d3ec0b87d85a1be62325b0
URL: https://github.com/llvm/llvm-project/commit/8bba070ef84ce41be4d3ec0b87d85a1be62325b0
DIFF: https://github.com/llvm/llvm-project/commit/8bba070ef84ce41be4d3ec0b87d85a1be62325b0.diff
LOG: AMDGPU: Expand testing of atomicrmw fmin/fmax lowering
Cover amdgpu.no.fine.grained.memory vs. amdgpu.no.remote.memory.
Added:
Modified:
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 46e571e7c5bc7..06dee9c279f2c 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -13,8 +13,8 @@
; float
; --------------------------------------------------------------------
-define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -28,7 +28,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -57,7 +57,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s4
@@ -68,7 +68,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -79,7 +79,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -107,7 +107,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -136,7 +136,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -165,7 +165,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
@@ -174,7 +174,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s8
@@ -184,12 +184,12 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -203,7 +203,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, s4
@@ -231,7 +231,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s4
@@ -242,7 +242,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -253,7 +253,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s8
@@ -280,7 +280,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s8
@@ -308,7 +308,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s8
@@ -336,7 +336,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
@@ -345,7 +345,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s8
@@ -355,12 +355,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr addrspace(7) %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -393,7 +393,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
@@ -454,7 +454,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX940-NEXT: v_mov_b32_e32 v0, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_mov_b32 s1, exec_lo
@@ -484,7 +484,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_mov_b32 s5, exec_lo
@@ -513,7 +513,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
@@ -572,7 +572,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
@@ -632,7 +632,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
@@ -692,7 +692,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[6:7], exec
@@ -718,7 +718,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b64 s[6:7], exec
@@ -745,144 +745,126 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall(ptr ad
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-; --------------------------------------------------------------------
-; double
-; --------------------------------------------------------------------
-
-define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset(ptr addrspace(7) inreg %ptr, double %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s4
-; GFX12-NEXT: s_addk_co_i32 s4, 0x800
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_mov_b32_e32 v6, s4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB3_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: v_mov_b32_e32 v1, s4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
-; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
-; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX940-NEXT: s_add_i32 s6, s4, 0x400
+; GFX940-NEXT: s_mov_b64 s[4:5], 0
+; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX940-NEXT: v_mov_b32_e32 v3, s6
+; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v5, v0
+; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
+; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX940-NEXT: s_cbranch_execnz .LBB3_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s4
-; GFX11-NEXT: s_addk_i32 s4, 0x800
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
-; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_max_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB3_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v0, s8
+; GFX90A-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX90A-NEXT: s_add_i32 s10, s8, 0x400
+; GFX90A-NEXT: s_mov_b64 s[8:9], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s10
+; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s8
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX908-NEXT: s_add_i32 s10, s8, 0x400
; GFX908-NEXT: s_mov_b64 s[8:9], 0
-; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s10
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v10, v1
-; GFX908-NEXT: v_mov_b32_e32 v9, v0
-; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v7
-; GFX908-NEXT: v_mov_b32_e32 v1, v8
-; GFX908-NEXT: v_mov_b32_e32 v2, v9
-; GFX908-NEXT: v_mov_b32_e32 v3, v10
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
@@ -890,32 +872,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX8-NEXT: s_add_i32 s10, s8, 0x400
; GFX8-NEXT: s_mov_b64 s[8:9], 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s10
; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v10, v1
-; GFX8-NEXT: v_mov_b32_e32 v9, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v7
-; GFX8-NEXT: v_mov_b32_e32 v1, v8
-; GFX8-NEXT: v_mov_b32_e32 v2, v9
-; GFX8-NEXT: v_mov_b32_e32 v3, v10
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
@@ -923,635 +901,1393 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX7-NEXT: v_mov_b32_e32 v1, s8
+; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX6-NEXT: v_mov_b32_e32 v1, s8
+; GFX6-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret float %result
}
-define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset(ptr addrspace(7) inreg %ptr, double %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
-; GFX12-NEXT: s_addk_co_i32 s4, 0x800
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v6, s4
-; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB4_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: v_mov_b32_e32 v1, s4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
-; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
-; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX940-NEXT: s_add_i32 s6, s4, 0x400
+; GFX940-NEXT: s_mov_b64 s[4:5], 0
+; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX940-NEXT: v_mov_b32_e32 v3, s6
+; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v5, v0
+; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
+; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX940-NEXT: s_cbranch_execnz .LBB4_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX11-NEXT: s_addk_i32 s4, 0x800
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
-; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_max_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
-; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB4_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v0, s8
+; GFX90A-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX90A-NEXT: s_add_i32 s10, s8, 0x400
+; GFX90A-NEXT: s_mov_b64 s[8:9], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s10
+; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX908-NEXT: s_add_i32 s10, s8, 0x400
; GFX908-NEXT: s_mov_b64 s[8:9], 0
-; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s10
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, v3
-; GFX908-NEXT: v_mov_b32_e32 v9, v2
-; GFX908-NEXT: v_mov_b32_e32 v8, v1
-; GFX908-NEXT: v_mov_b32_e32 v7, v0
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
-; GFX908-NEXT: v_mov_b32_e32 v2, v7
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v3, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
-; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
-; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX8-NEXT: s_add_i32 s10, s8, 0x400
; GFX8-NEXT: s_mov_b64 s[8:9], 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s10
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, v3
-; GFX8-NEXT: v_mov_b32_e32 v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v8, v1
-; GFX8-NEXT: v_mov_b32_e32 v7, v0
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v2, v7
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX7-NEXT: v_mov_b32_e32 v1, s8
+; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX6-NEXT: v_mov_b32_e32 v1, s8
+; GFX6-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmax ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret float %result
}
-define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall(ptr addrspace(7) %ptr, double %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; --------------------------------------------------------------------
+; double
+; --------------------------------------------------------------------
+
+define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
-; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB5_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[5:6], v[5:6]
-; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB5_4 Depth 2
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[11:12], v[0:1], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
-; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
-; GFX12-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB5_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
+; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
-; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB5_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v7, v6
-; GFX940-NEXT: v_mov_b32_e32 v6, v5
-; GFX940-NEXT: s_mov_b64 s[2:3], exec
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX940-NEXT: v_readfirstlane_b32 s4, v0
-; GFX940-NEXT: v_readfirstlane_b32 s5, v1
-; GFX940-NEXT: v_readfirstlane_b32 s6, v2
-; GFX940-NEXT: v_readfirstlane_b32 s7, v3
-; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
-; GFX940-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX940-NEXT: ; implicit-def: $vgpr4
-; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB5_1
-; GFX940-NEXT: ; %bb.2:
-; GFX940-NEXT: s_mov_b64 exec, s[2:3]
+; GFX940-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v0, v6
-; GFX940-NEXT: v_mov_b32_e32 v1, v7
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
-; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB5_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
-; GFX11-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB5_4 Depth 2
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[11:12], v[0:1], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
-; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
-; GFX11-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], 0 offen glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB5_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
-; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB5_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB5_1
-; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, v5
-; GFX10-NEXT: v_mov_b32_e32 v1, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v7, v6
-; GFX90A-NEXT: v_mov_b32_e32 v6, v5
-; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
-; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
-; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX90A-NEXT: ; implicit-def: $vgpr4
-; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
-; GFX90A-NEXT: ; %bb.2:
-; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v0, v6
-; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v8, v3
-; GFX908-NEXT: v_mov_b32_e32 v7, v2
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
+; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
-; GFX908-NEXT: v_add_u32_e32 v15, 0x800, v4
-; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_readfirstlane_b32 s8, v9
-; GFX908-NEXT: v_readfirstlane_b32 s9, v10
-; GFX908-NEXT: v_readfirstlane_b32 s10, v7
-; GFX908-NEXT: v_readfirstlane_b32 s11, v8
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
-; GFX908-NEXT: ; implicit-def: $vgpr4
-; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB5_1
-; GFX908-NEXT: ; %bb.2:
-; GFX908-NEXT: s_mov_b64 exec, s[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB5_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX908-NEXT: s_mov_b64 s[12:13], exec
-; GFX908-NEXT: v_max_f64 v[11:12], v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v11
-; GFX908-NEXT: v_mov_b32_e32 v1, v12
-; GFX908-NEXT: v_mov_b32_e32 v2, v13
-; GFX908-NEXT: v_mov_b32_e32 v3, v14
-; GFX908-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT: v_readfirstlane_b32 s8, v9
-; GFX908-NEXT: v_readfirstlane_b32 s9, v10
-; GFX908-NEXT: v_readfirstlane_b32 s10, v7
-; GFX908-NEXT: v_readfirstlane_b32 s11, v8
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
-; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB5_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX908-NEXT: s_mov_b64 exec, s[12:13]
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v7
+; GFX908-NEXT: v_mov_b32_e32 v1, v8
+; GFX908-NEXT: v_mov_b32_e32 v2, v9
+; GFX908-NEXT: v_mov_b32_e32 v3, v10
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
-; GFX908-NEXT: v_mov_b32_e32 v14, v1
-; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v13, v0
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB5_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB5_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v8, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
-; GFX8-NEXT: v_add_u32_e32 v15, vcc, 0x800, v4
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_readfirstlane_b32 s8, v9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v10
-; GFX8-NEXT: v_readfirstlane_b32 s10, v7
-; GFX8-NEXT: v_readfirstlane_b32 s11, v8
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
-; GFX8-NEXT: ; implicit-def: $vgpr4
-; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB5_1
-; GFX8-NEXT: ; %bb.2:
-; GFX8-NEXT: s_mov_b64 exec, s[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB5_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX8-NEXT: s_mov_b64 s[12:13], exec
-; GFX8-NEXT: v_max_f64 v[11:12], v[0:1], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v11
-; GFX8-NEXT: v_mov_b32_e32 v1, v12
-; GFX8-NEXT: v_mov_b32_e32 v2, v13
-; GFX8-NEXT: v_mov_b32_e32 v3, v14
-; GFX8-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX8-NEXT: v_readfirstlane_b32 s8, v9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v10
-; GFX8-NEXT: v_readfirstlane_b32 s10, v7
-; GFX8-NEXT: v_readfirstlane_b32 s11, v8
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
-; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB5_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX8-NEXT: s_mov_b64 exec, s[12:13]
+; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
-; GFX8-NEXT: v_mov_b32_e32 v14, v1
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v13, v0
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB5_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB5_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_readfirstlane_b32 s8, v0
-; GFX7-NEXT: v_readfirstlane_b32 s9, v1
-; GFX7-NEXT: v_readfirstlane_b32 s10, v2
-; GFX7-NEXT: v_readfirstlane_b32 s11, v3
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX7-NEXT: ; implicit-def: $vgpr4
-; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB5_1
-; GFX7-NEXT: ; %bb.2:
-; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, v5
-; GFX7-NEXT: v_mov_b32_e32 v1, v6
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_readfirstlane_b32 s8, v0
-; GFX6-NEXT: v_readfirstlane_b32 s9, v1
-; GFX6-NEXT: v_readfirstlane_b32 s10, v2
-; GFX6-NEXT: v_readfirstlane_b32 s11, v3
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX6-NEXT: ; implicit-def: $vgpr4
-; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB5_1
-; GFX6-NEXT: ; %bb.2:
-; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v10, v3
+; GFX908-NEXT: v_mov_b32_e32 v9, v2
+; GFX908-NEXT: v_mov_b32_e32 v8, v1
+; GFX908-NEXT: v_mov_b32_e32 v7, v0
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v2, v7
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: v_mov_b32_e32 v3, v8
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB6_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v10, v3
+; GFX8-NEXT: v_mov_b32_e32 v9, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, v7
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: v_mov_b32_e32 v3, v8
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB6_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %unused = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
+; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_readfirstlane_b32 s4, v9
+; GFX12-NEXT: v_readfirstlane_b32 s5, v10
+; GFX12-NEXT: v_readfirstlane_b32 s6, v7
+; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
+; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-NEXT: ; %bb.2:
+; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[5:6], v[5:6]
+; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
+; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[11:12], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
+; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
+; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-NEXT: v_readfirstlane_b32 s4, v9
+; GFX12-NEXT: v_readfirstlane_b32 s5, v10
+; GFX12-NEXT: v_readfirstlane_b32 s6, v7
+; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB7_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
+; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_cbranch_execnz .LBB7_3
+; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v7, v6
+; GFX940-NEXT: v_mov_b32_e32 v6, v5
+; GFX940-NEXT: s_mov_b64 s[2:3], exec
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: v_readfirstlane_b32 s4, v0
+; GFX940-NEXT: v_readfirstlane_b32 s5, v1
+; GFX940-NEXT: v_readfirstlane_b32 s6, v2
+; GFX940-NEXT: v_readfirstlane_b32 s7, v3
+; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
+; GFX940-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX940-NEXT: ; implicit-def: $vgpr4
+; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB7_1
+; GFX940-NEXT: ; %bb.2:
+; GFX940-NEXT: s_mov_b64 exec, s[2:3]
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v6
+; GFX940-NEXT: v_mov_b32_e32 v1, v7
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
+; GFX11-NEXT: s_mov_b32 s1, 0
+; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-NEXT: v_readfirstlane_b32 s5, v10
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: v_readfirstlane_b32 s7, v8
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
+; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
+; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[11:12], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
+; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
+; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-NEXT: v_readfirstlane_b32 s5, v10
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: v_readfirstlane_b32 s7, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], 0 offen glc
+; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB7_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
+; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_cbranch_execnz .LBB7_3
+; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s8, v0
+; GFX10-NEXT: v_readfirstlane_b32 s9, v1
+; GFX10-NEXT: v_readfirstlane_b32 s10, v2
+; GFX10-NEXT: v_readfirstlane_b32 s11, v3
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
+; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
+; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB7_1
+; GFX10-NEXT: ; %bb.2:
+; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v7, v6
+; GFX90A-NEXT: v_mov_b32_e32 v6, v5
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
+; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
+; GFX90A-NEXT: ; %bb.2:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v0, v6
+; GFX90A-NEXT: v_mov_b32_e32 v1, v7
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v8, v3
+; GFX908-NEXT: v_mov_b32_e32 v7, v2
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_add_u32_e32 v15, 0x800, v4
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v9
+; GFX908-NEXT: v_readfirstlane_b32 s9, v10
+; GFX908-NEXT: v_readfirstlane_b32 s10, v7
+; GFX908-NEXT: v_readfirstlane_b32 s11, v8
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
+; GFX908-NEXT: ; implicit-def: $vgpr4
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB7_1
+; GFX908-NEXT: ; %bb.2:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Loop Header: Depth=1
+; GFX908-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
+; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_max_f64 v[11:12], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v11
+; GFX908-NEXT: v_mov_b32_e32 v1, v12
+; GFX908-NEXT: v_mov_b32_e32 v2, v13
+; GFX908-NEXT: v_mov_b32_e32 v3, v14
+; GFX908-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX908-NEXT: v_readfirstlane_b32 s8, v9
+; GFX908-NEXT: v_readfirstlane_b32 s9, v10
+; GFX908-NEXT: v_readfirstlane_b32 s10, v7
+; GFX908-NEXT: v_readfirstlane_b32 s11, v8
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB7_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX908-NEXT: s_mov_b64 exec, s[12:13]
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
+; GFX908-NEXT: v_mov_b32_e32 v14, v1
+; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v13, v0
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_cbranch_execnz .LBB7_3
+; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v8, v3
+; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, 0x800, v4
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v9
+; GFX8-NEXT: v_readfirstlane_b32 s9, v10
+; GFX8-NEXT: v_readfirstlane_b32 s10, v7
+; GFX8-NEXT: v_readfirstlane_b32 s11, v8
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
+; GFX8-NEXT: ; implicit-def: $vgpr4
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_1
+; GFX8-NEXT: ; %bb.2:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Loop Header: Depth=1
+; GFX8-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
+; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_max_f64 v[11:12], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v11
+; GFX8-NEXT: v_mov_b32_e32 v1, v12
+; GFX8-NEXT: v_mov_b32_e32 v2, v13
+; GFX8-NEXT: v_mov_b32_e32 v3, v14
+; GFX8-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX8-NEXT: v_readfirstlane_b32 s8, v9
+; GFX8-NEXT: v_readfirstlane_b32 s9, v10
+; GFX8-NEXT: v_readfirstlane_b32 s10, v7
+; GFX8-NEXT: v_readfirstlane_b32 s11, v8
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX8-NEXT: s_mov_b64 exec, s[12:13]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
+; GFX8-NEXT: v_mov_b32_e32 v14, v1
+; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v13, v0
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_3
+; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr4
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB7_1
+; GFX7-NEXT: ; %bb.2:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, v5
+; GFX7-NEXT: v_mov_b32_e32 v1, v6
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB7_1
+; GFX6-NEXT: ; %bb.2:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, v5
+; GFX6-NEXT: v_mov_b32_e32 v1, v6
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
+; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
+; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v7
+; GFX908-NEXT: v_mov_b32_e32 v1, v8
+; GFX908-NEXT: v_mov_b32_e32 v2, v9
+; GFX908-NEXT: v_mov_b32_e32 v3, v10
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB8_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB8_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret double %result
+}
+
+define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
+; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB9_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
+; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v7
+; GFX908-NEXT: v_mov_b32_e32 v1, v8
+; GFX908-NEXT: v_mov_b32_e32 v2, v9
+; GFX908-NEXT: v_mov_b32_e32 v3, v10
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB9_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB9_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v5
-; GFX6-NEXT: v_mov_b32_e32 v1, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret double %result
}
@@ -1559,8 +2295,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall(ptr a
; half
; --------------------------------------------------------------------
-define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) inreg %ptr, half %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1578,7 +2314,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -1600,13 +2336,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -1619,7 +2355,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX940-NEXT: s_not_b32 s7, s4
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX940-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v1
@@ -1636,13 +2372,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB6_1
+; GFX940-NEXT: s_cbranch_execnz .LBB10_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -1657,7 +2393,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -1680,13 +2416,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -1699,7 +2435,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v4, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v1
@@ -1718,13 +2454,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB6_1
+; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -1737,7 +2473,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX90A-NEXT: s_not_b32 s11, s8
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1753,13 +2489,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -1772,7 +2508,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX908-NEXT: s_not_b32 s11, s8
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1789,13 +2525,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB6_1
+; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -1808,7 +2544,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1826,13 +2562,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB6_1
+; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -1846,7 +2582,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1865,14 +2601,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB6_1
+; GFX7-NEXT: s_cbranch_execnz .LBB10_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -1886,7 +2622,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1906,7 +2642,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB6_1
+; GFX6-NEXT: s_cbranch_execnz .LBB10_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v2
@@ -1914,12 +2650,12 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset(ptr addrspace(7) i
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7) inreg %ptr, half %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1937,7 +2673,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -1959,12 +2695,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -1977,7 +2713,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX940-NEXT: s_not_b32 s7, s4
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX940-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v1
@@ -1994,12 +2730,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB7_1
+; GFX940-NEXT: s_cbranch_execnz .LBB11_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -2014,7 +2750,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -2037,12 +2773,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -2055,7 +2791,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v2, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v1
@@ -2074,12 +2810,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB7_1
+; GFX10-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -2092,7 +2828,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX90A-NEXT: s_not_b32 s11, s8
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2108,12 +2844,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -2126,7 +2862,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX908-NEXT: s_not_b32 s11, s8
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2143,12 +2879,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB7_1
+; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -2161,7 +2897,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2179,12 +2915,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB7_1
+; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -2198,7 +2934,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2217,12 +2953,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB7_1
+; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -2236,7 +2972,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2256,18 +2992,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset(ptr addrspace(7)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB7_1
+; GFX6-NEXT: s_cbranch_execnz .LBB11_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmax ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr addrspace(7) %ptr, half %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, half %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2283,7 +3019,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -2296,14 +3032,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_max_num_f16_e32 v10, v5, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-NEXT: s_mov_b32 s2, exec_lo
@@ -2318,7 +3054,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX12-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -2333,8 +3069,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB8_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB12_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -2343,13 +3079,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB8_3
+; GFX12-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -2360,7 +3096,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: v_lshlrev_b32_e64 v4, v8, s0
; GFX940-NEXT: v_not_b32_e32 v10, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -2372,14 +3108,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX940-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB8_1
+; GFX940-NEXT: s_cbranch_execnz .LBB12_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
; GFX940-NEXT: v_max_f16_e32 v11, v5, v5
-; GFX940-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB12_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v4, v8, v7
; GFX940-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2389,7 +3125,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: s_mov_b64 s[8:9], exec
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX940-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -2403,8 +3139,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB8_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB12_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -2412,13 +3148,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB8_3
+; GFX940-NEXT: s_cbranch_execnz .LBB12_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -2431,7 +3167,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -2444,14 +3180,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-NEXT: s_mov_b32 s2, exec_lo
@@ -2466,7 +3202,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX11-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -2481,8 +3217,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB8_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB12_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -2492,13 +3228,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB8_3
+; GFX11-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -2509,7 +3245,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
-; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -2521,13 +3257,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB8_1
+; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
-; GFX10-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX10-NEXT: s_mov_b32 s6, exec_lo
@@ -2538,7 +3274,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -2552,8 +3288,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB8_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB12_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -2563,13 +3299,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB8_3
+; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -2580,7 +3316,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
; GFX90A-NEXT: v_not_b32_e32 v10, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -2592,14 +3328,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_max_f16_e32 v11, v5, v5
-; GFX90A-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB12_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v8, v7
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2608,7 +3344,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX90A-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -2621,8 +3357,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB8_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -2630,13 +3366,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB8_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB12_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -2647,7 +3383,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX908-NEXT: v_not_b32_e32 v9, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -2659,14 +3395,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB8_1
+; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_max_f16_e32 v10, v5, v5
-; GFX908-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB12_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX908-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2676,7 +3412,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX908-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -2689,8 +3425,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB8_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB12_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2698,13 +3434,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB8_3
+; GFX908-NEXT: s_cbranch_execnz .LBB12_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
@@ -2715,7 +3451,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX8-NEXT: v_not_b32_e32 v9, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -2727,14 +3463,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB8_1
+; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_max_f16_e32 v10, v5, v5
-; GFX8-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB12_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX8-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2745,7 +3481,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX8-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -2758,8 +3494,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB8_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB12_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2767,13 +3503,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB8_3
+; GFX8-NEXT: s_cbranch_execnz .LBB12_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -2783,7 +3519,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -2794,15 +3530,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB8_1
+; GFX7-NEXT: s_cbranch_execnz .LBB12_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v4
-; GFX7-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB12_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
@@ -2814,7 +3550,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
; GFX7-NEXT: v_mov_b32_e32 v4, v5
; GFX7-NEXT: v_mov_b32_e32 v5, v6
-; GFX7-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX7-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -2827,8 +3563,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB8_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB12_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2836,14 +3572,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB8_3
+; GFX7-NEXT: s_cbranch_execnz .LBB12_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -2853,7 +3589,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -2864,15 +3600,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB8_1
+; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v4
-; GFX6-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB12_4 Depth 2
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
@@ -2884,7 +3620,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
; GFX6-NEXT: v_mov_b32_e32 v4, v5
; GFX6-NEXT: v_mov_b32_e32 v5, v6
-; GFX6-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX6-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -2897,8 +3633,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB8_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB12_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2906,7 +3642,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB8_3
+; GFX6-NEXT: s_cbranch_execnz .LBB12_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
@@ -2914,7 +3650,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
@@ -2922,8 +3658,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall(ptr add
; bfloat
; --------------------------------------------------------------------
-define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2941,7 +3677,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -2970,13 +3706,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB9_1
+; GFX12-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -2990,7 +3726,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX940-NEXT: s_movk_i32 s8, 0x7fff
-; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3012,13 +3748,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB9_1
+; GFX940-NEXT: s_cbranch_execnz .LBB13_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -3033,7 +3769,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -3063,13 +3799,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -3082,7 +3818,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v4, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3105,13 +3841,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB9_1
+; GFX10-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -3125,7 +3861,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX90A-NEXT: s_movk_i32 s12, 0x7fff
-; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3145,13 +3881,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -3165,7 +3901,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX908-NEXT: s_movk_i32 s12, 0x7fff
-; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3186,13 +3922,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB9_1
+; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -3205,7 +3941,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v0, s10
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -3229,13 +3965,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB9_1
+; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -3249,7 +3985,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3269,14 +4005,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB9_1
+; GFX7-NEXT: s_cbranch_execnz .LBB13_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -3290,7 +4026,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3311,7 +4047,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB9_1
+; GFX6-NEXT: s_cbranch_execnz .LBB13_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v2
@@ -3319,12 +4055,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3342,7 +4078,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -3371,12 +4107,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB10_1
+; GFX12-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -3390,7 +4126,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX940-NEXT: s_movk_i32 s8, 0x7fff
-; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3412,12 +4148,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB10_1
+; GFX940-NEXT: s_cbranch_execnz .LBB14_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -3432,7 +4168,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -3462,12 +4198,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB10_1
+; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -3480,7 +4216,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v2, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3503,12 +4239,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB10_1
+; GFX10-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -3522,7 +4258,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s12, 0x7fff
-; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3542,12 +4278,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -3561,7 +4297,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX908-NEXT: s_movk_i32 s12, 0x7fff
-; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3582,12 +4318,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB10_1
+; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -3600,7 +4336,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v0, s10
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -3624,12 +4360,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB10_1
+; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -3643,7 +4379,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3663,12 +4399,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB10_1
+; GFX7-NEXT: s_cbranch_execnz .LBB14_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -3682,7 +4418,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3703,18 +4439,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB10_1
+; GFX6-NEXT: s_cbranch_execnz .LBB14_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmax ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr addrspace(7) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3730,7 +4466,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -3743,14 +4479,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-NEXT: s_mov_b32 s2, exec_lo
@@ -3772,7 +4508,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX12-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -3787,8 +4523,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB11_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB15_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -3797,13 +4533,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB11_3
+; GFX12-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -3814,7 +4550,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: v_lshlrev_b32_e64 v4, v8, s0
; GFX940-NEXT: v_not_b32_e32 v10, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -3826,15 +4562,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX940-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB11_1
+; GFX940-NEXT: s_cbranch_execnz .LBB15_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v11, 16, v5
; GFX940-NEXT: s_movk_i32 s10, 0x7fff
-; GFX940-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB15_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX940-NEXT: s_mov_b64 s[8:9], exec
@@ -3849,7 +4585,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX940-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
-; GFX940-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX940-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -3863,8 +4599,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB11_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB15_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -3872,13 +4608,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB11_3
+; GFX940-NEXT: s_cbranch_execnz .LBB15_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -3891,7 +4627,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -3904,15 +4640,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB11_1
+; GFX11-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-NEXT: s_mov_b32 s2, exec_lo
@@ -3934,7 +4670,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX11-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -3949,8 +4685,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB11_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB15_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -3960,14 +4696,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB11_3
+; GFX11-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -3978,7 +4714,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
-; GFX10-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -3990,13 +4726,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB11_1
+; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX10-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_mov_b32 s6, exec_lo
@@ -4011,7 +4747,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -4025,8 +4761,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB11_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB15_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -4036,13 +4772,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB11_3
+; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -4053,7 +4789,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
; GFX90A-NEXT: v_not_b32_e32 v10, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -4065,15 +4801,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v11, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
-; GFX90A-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB15_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_max_f32_e32 v4, v4, v11
@@ -4086,7 +4822,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX90A-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -4099,8 +4835,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB11_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -4108,13 +4844,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB11_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -4125,7 +4861,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX908-NEXT: v_not_b32_e32 v9, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -4137,15 +4873,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB11_1
+; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
-; GFX908-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB15_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_max_f32_e32 v4, v4, v10
@@ -4159,7 +4895,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX908-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -4172,8 +4908,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB11_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB15_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4181,13 +4917,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB11_3
+; GFX908-NEXT: s_cbranch_execnz .LBB15_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
@@ -4198,7 +4934,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX8-NEXT: v_not_b32_e32 v9, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -4210,14 +4946,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB11_1
+; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX8-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB15_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_max_f32_e32 v4, v4, v10
@@ -4233,7 +4969,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX8-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -4246,8 +4982,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB11_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB15_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4255,13 +4991,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB11_3
+; GFX8-NEXT: s_cbranch_execnz .LBB15_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -4271,7 +5007,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -4282,15 +5018,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB11_1
+; GFX7-NEXT: s_cbranch_execnz .LBB15_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
-; GFX7-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB15_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
@@ -4303,7 +5039,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: v_mov_b32_e32 v4, v5
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v5, v6
-; GFX7-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX7-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -4316,8 +5052,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB11_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB15_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4325,14 +5061,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB11_3
+; GFX7-NEXT: s_cbranch_execnz .LBB15_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -4342,7 +5078,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -4353,15 +5089,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB11_1
+; GFX6-NEXT: s_cbranch_execnz .LBB15_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
-; GFX6-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB15_4 Depth 2
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
@@ -4374,7 +5110,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: v_mov_b32_e32 v4, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v5, v6
-; GFX6-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX6-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -4387,8 +5123,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB11_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB15_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4396,7 +5132,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB11_3
+; GFX6-NEXT: s_cbranch_execnz .LBB15_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
@@ -4404,7 +5140,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
@@ -4412,8 +5148,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall(ptr
; <2 x half>
; --------------------------------------------------------------------
-define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4427,7 +5163,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v5, v0
@@ -4444,12 +5180,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -4459,7 +5195,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_pk_max_f16 v2, v1, v1
; GFX940-NEXT: v_mov_b32_e32 v3, s6
-; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v0
@@ -4474,12 +5210,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB12_1
+; GFX940-NEXT: s_cbranch_execnz .LBB16_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
@@ -4489,7 +5225,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v5, v0
@@ -4507,12 +5243,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB12_1
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, v0
@@ -4522,7 +5258,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -4538,12 +5274,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB12_1
+; GFX10-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -4553,7 +5289,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: v_mov_b32_e32 v3, s10
-; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
@@ -4566,12 +5302,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -4581,7 +5317,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_pk_max_f16 v2, v1, v1
; GFX908-NEXT: v_mov_b32_e32 v3, s10
-; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
@@ -4595,12 +5331,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB12_1
+; GFX908-NEXT: s_cbranch_execnz .LBB16_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -4611,7 +5347,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v3, v1, v1
; GFX8-NEXT: v_mov_b32_e32 v4, s10
-; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -4628,12 +5364,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB12_1
+; GFX8-NEXT: s_cbranch_execnz .LBB16_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -4649,7 +5385,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_mov_b32_e32 v4, s10
-; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -4674,12 +5410,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB12_1
+; GFX7-NEXT: s_cbranch_execnz .LBB16_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -4695,7 +5431,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX6-NEXT: v_mov_b32_e32 v4, s10
-; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -4721,18 +5457,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset(ptr addrsp
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB12_1
+; GFX6-NEXT: s_cbranch_execnz .LBB16_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4745,7 +5481,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX12-NEXT: v_mov_b32_e32 v3, s4
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v0, v1, v1
@@ -4761,12 +5497,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, s4
@@ -4775,7 +5511,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_pk_max_f16 v2, v0, v0
; GFX940-NEXT: v_mov_b32_e32 v3, s6
-; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4790,12 +5526,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB13_1
+; GFX940-NEXT: s_cbranch_execnz .LBB17_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s4
@@ -4804,7 +5540,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX11-NEXT: v_mov_b32_e32 v3, s4
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4821,12 +5557,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -4835,7 +5571,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, s8
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: buffer_load_dword v1, v1, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4851,12 +5587,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB13_1
+; GFX10-NEXT: s_cbranch_execnz .LBB17_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s8
@@ -4865,7 +5601,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s10
-; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4878,12 +5614,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s8
@@ -4892,7 +5628,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s10
-; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4906,12 +5642,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB13_1
+; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s8
@@ -4921,7 +5657,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s10
-; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -4938,12 +5674,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB13_1
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -4959,7 +5695,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v5
; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
@@ -4984,12 +5720,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB13_1
+; GFX7-NEXT: s_cbranch_execnz .LBB17_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -5005,7 +5741,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v5
; GFX6-NEXT: v_mov_b32_e32 v2, s10
-; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
@@ -5031,18 +5767,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset(ptr addrspace(
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB13_1
+; GFX6-NEXT: s_cbranch_execnz .LBB17_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmax ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5051,7 +5787,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -5065,14 +5801,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v4, v6, v6
; GFX12-NEXT: s_mov_b32 s2, exec_lo
@@ -5081,7 +5817,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v8
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -5096,8 +5832,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB18_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -5106,18 +5842,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB14_3
+; GFX12-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -5130,21 +5866,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX940-NEXT: ; implicit-def: $vgpr4
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB14_1
+; GFX940-NEXT: s_cbranch_execnz .LBB18_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
; GFX940-NEXT: v_pk_max_f16 v9, v5, v5
-; GFX940-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB18_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v4, v7, v7
; GFX940-NEXT: s_mov_b64 s[8:9], exec
; GFX940-NEXT: v_pk_max_f16 v6, v4, v9
; GFX940-NEXT: buffer_wbl2 sc1
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
-; GFX940-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX940-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -5158,8 +5894,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB14_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB18_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -5167,19 +5903,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB14_3
+; GFX940-NEXT: s_cbranch_execnz .LBB18_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -5193,14 +5929,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: buffer_load_b32 v6, v4, s[4:7], 0 offen offset:1024
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v4, v6, v6
; GFX11-NEXT: s_mov_b32 s2, exec_lo
@@ -5209,7 +5945,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: v_pk_max_f16 v5, v4, v8
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -5224,8 +5960,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB18_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -5235,19 +5971,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB14_3
+; GFX11-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -5260,13 +5996,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: ; implicit-def: $vgpr4
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_1
+; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX10-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v4, v6, v6
; GFX10-NEXT: s_mov_b32 s6, exec_lo
@@ -5274,7 +6010,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_pk_max_f16 v5, v4, v8
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -5288,8 +6024,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB18_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -5299,18 +6035,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB14_3
+; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -5323,20 +6059,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_pk_max_f16 v9, v5, v5
-; GFX90A-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB18_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v4, v7, v7
; GFX90A-NEXT: v_pk_max_f16 v6, v4, v9
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX90A-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -5349,8 +6085,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB14_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -5358,18 +6094,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB14_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -5382,21 +6118,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: ; implicit-def: $vgpr4
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB14_1
+; GFX908-NEXT: s_cbranch_execnz .LBB18_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX908-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB18_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v4, v6, v6
; GFX908-NEXT: v_pk_max_f16 v5, v4, v8
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX908-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -5409,8 +6145,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB14_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB18_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -5418,18 +6154,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB14_3
+; GFX908-NEXT: s_cbranch_execnz .LBB18_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -5442,15 +6178,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: ; implicit-def: $vgpr4
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB14_1
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
-; GFX8-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB18_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v4, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
@@ -5460,7 +6196,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX8-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -5473,8 +6209,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB14_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB18_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -5482,18 +6218,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB14_3
+; GFX8-NEXT: s_cbranch_execnz .LBB18_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -5505,7 +6241,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_1
+; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -5517,9 +6253,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v11, v8
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB18_4 Depth 2
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[12:13], exec
@@ -5535,7 +6271,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
; GFX7-NEXT: v_mov_b32_e32 v8, v6
; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -5548,8 +6284,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB18_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
@@ -5559,19 +6295,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_3
+; GFX7-NEXT: s_cbranch_execnz .LBB18_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: v_mov_b32_e32 v1, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -5583,7 +6319,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_1
+; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -5595,9 +6331,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v11, v8
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB18_4 Depth 2
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
@@ -5614,7 +6350,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
; GFX6-NEXT: v_mov_b32_e32 v8, v6
; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -5627,8 +6363,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB18_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
@@ -5638,7 +6374,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_3
+; GFX6-NEXT: s_cbranch_execnz .LBB18_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v4
@@ -5646,7 +6382,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
@@ -5654,8 +6390,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; <2 x bfloat>
; --------------------------------------------------------------------
-define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5670,7 +6406,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
; GFX12-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v0
@@ -5703,12 +6439,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-NEXT: s_cbranch_execnz .LBB19_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -5721,7 +6457,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX940-NEXT: s_mov_b32 s9, 0x7060302
; GFX940-NEXT: v_mov_b32_e32 v4, s4
-; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v0
@@ -5748,12 +6484,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX940-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX940-NEXT: s_cbranch_execnz .LBB15_1
+; GFX940-NEXT: s_cbranch_execnz .LBB19_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
@@ -5766,7 +6502,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v0
@@ -5800,13 +6536,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-NEXT: s_cbranch_execnz .LBB19_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, v0
@@ -5817,7 +6553,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -5846,12 +6582,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB15_1
+; GFX10-NEXT: s_cbranch_execnz .LBB19_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -5864,7 +6600,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s13, 0x7060302
; GFX90A-NEXT: v_mov_b32_e32 v4, s8
-; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
@@ -5890,12 +6626,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -5908,7 +6644,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s13, 0x7060302
; GFX908-NEXT: v_mov_b32_e32 v4, s8
-; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
@@ -5935,12 +6671,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX908-NEXT: s_cbranch_execnz .LBB15_1
+; GFX908-NEXT: s_cbranch_execnz .LBB19_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -5951,7 +6687,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -5981,12 +6717,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_cbranch_execnz .LBB15_1
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -6001,7 +6737,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v4, s8
-; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -6023,12 +6759,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX7-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX7-NEXT: s_cbranch_execnz .LBB15_1
+; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -6043,7 +6779,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v4, s8
-; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -6066,18 +6802,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset(ptr add
; GFX6-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX6-NEXT: s_cbranch_execnz .LBB15_1
+; GFX6-NEXT: s_cbranch_execnz .LBB19_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6090,7 +6826,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v0
; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
@@ -6121,12 +6857,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: s_cbranch_execnz .LBB20_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, s4
@@ -6138,7 +6874,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX940-NEXT: s_mov_b32 s9, 0x7060302
; GFX940-NEXT: v_mov_b32_e32 v4, s4
-; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6165,12 +6901,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX940-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX940-NEXT: v_mov_b32_e32 v1, v6
; GFX940-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX940-NEXT: s_cbranch_execnz .LBB16_1
+; GFX940-NEXT: s_cbranch_execnz .LBB20_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_lshlrev_b32 v2, 16, v0
@@ -6181,7 +6917,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
@@ -6213,13 +6949,13 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: s_cbranch_execnz .LBB20_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -6229,7 +6965,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v4, s8
; GFX10-NEXT: buffer_load_dword v1, v1, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6258,12 +6994,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB16_1
+; GFX10-NEXT: s_cbranch_execnz .LBB20_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s8
@@ -6275,7 +7011,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s13, 0x7060302
; GFX90A-NEXT: v_mov_b32_e32 v4, s8
-; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6301,12 +7037,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX90A-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s8
@@ -6318,7 +7054,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b32 s13, 0x7060302
; GFX908-NEXT: v_mov_b32_e32 v4, s8
-; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6345,12 +7081,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX908-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s8
@@ -6360,7 +7096,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6390,12 +7126,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -6410,7 +7146,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -6432,12 +7168,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX7-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX7-NEXT: s_cbranch_execnz .LBB16_1
+; GFX7-NEXT: s_cbranch_execnz .LBB20_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -6452,7 +7188,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -6475,18 +7211,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX6-NEXT: s_cbranch_execnz .LBB16_1
+; GFX6-NEXT: s_cbranch_execnz .LBB20_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall(ptr addrspace(7) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6495,7 +7231,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -6509,15 +7245,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-NEXT: buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v6
@@ -6541,7 +7277,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX12-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -6556,8 +7292,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -6566,18 +7302,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB17_3
+; GFX12-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -6590,7 +7326,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX940-NEXT: ; implicit-def: $vgpr4
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB17_1
+; GFX940-NEXT: s_cbranch_execnz .LBB21_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
@@ -6598,9 +7334,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX940-NEXT: s_movk_i32 s10, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
; GFX940-NEXT: s_mov_b32 s11, 0x7060302
-; GFX940-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB21_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX940-NEXT: v_max_f32_e32 v4, v4, v9
@@ -6621,7 +7357,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v11, vcc
; GFX940-NEXT: v_perm_b32 v6, v5, v4, s11
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
-; GFX940-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX940-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -6635,8 +7371,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB17_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB21_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -6644,19 +7380,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB17_3
+; GFX940-NEXT: s_cbranch_execnz .LBB21_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -6670,16 +7406,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-NEXT: buffer_load_b32 v6, v4, s[4:7], 0 offen offset:1024
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB21_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
@@ -6703,7 +7439,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -6718,8 +7454,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -6729,20 +7465,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB17_3
+; GFX11-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -6755,14 +7491,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: ; implicit-def: $vgpr4
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB17_1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
@@ -6783,7 +7519,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -6797,8 +7533,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB17_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -6808,18 +7544,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB17_3
+; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -6832,7 +7568,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
@@ -6840,9 +7576,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
-; GFX90A-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB21_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX90A-NEXT: v_max_f32_e32 v4, v4, v9
@@ -6861,7 +7597,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_perm_b32 v6, v5, v4, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX90A-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -6874,8 +7610,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB17_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -6883,18 +7619,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB17_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -6907,7 +7643,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: ; implicit-def: $vgpr4
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
@@ -6915,9 +7651,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
-; GFX908-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB21_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX908-NEXT: v_max_f32_e32 v4, v4, v8
@@ -6937,7 +7673,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -6950,8 +7686,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -6959,18 +7695,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_3
+; GFX908-NEXT: s_cbranch_execnz .LBB21_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -6983,15 +7719,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: ; implicit-def: $vgpr4
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX8-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB21_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX8-NEXT: v_max_f32_e32 v4, v4, v8
@@ -7014,7 +7750,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -7027,8 +7763,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -7036,18 +7772,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_3
+; GFX8-NEXT: s_cbranch_execnz .LBB21_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -7059,7 +7795,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
@@ -7070,9 +7806,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX7-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB21_4 Depth 2
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v7
@@ -7086,7 +7822,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_mov_b32_e32 v7, v5
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -7099,8 +7835,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB17_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
@@ -7109,19 +7845,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB17_3
+; GFX7-NEXT: s_cbranch_execnz .LBB21_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -7133,7 +7869,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB17_1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
@@ -7144,9 +7880,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX6-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB21_4 Depth 2
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v7
@@ -7160,7 +7896,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -7173,8 +7909,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB17_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
@@ -7184,14 +7920,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB17_3
+; GFX6-NEXT: s_cbranch_execnz .LBB21_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
@@ -7199,8 +7935,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; misc
; --------------------------------------------------------------------
-define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7213,7 +7949,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_max_num_f32 v2, v1, v1
; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v5, v0
@@ -7230,12 +7966,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: s_cbranch_execnz .LBB22_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX940-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -7245,7 +7981,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
; GFX940-NEXT: v_mov_b32_e32 v3, s6
-; GFX940-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v0
@@ -7259,12 +7995,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB18_1
+; GFX940-NEXT: s_cbranch_execnz .LBB22_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX11-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
@@ -7273,7 +8009,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_max_f32 v2, v1, v1
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v5, v0
@@ -7291,12 +8027,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: s_cbranch_execnz .LBB22_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, v0
@@ -7306,7 +8042,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: v_max_f32_e32 v2, v1, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -7322,12 +8058,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB18_1
+; GFX10-NEXT: s_cbranch_execnz .LBB22_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -7337,7 +8073,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
; GFX90A-NEXT: v_mov_b32_e32 v3, s10
-; GFX90A-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
@@ -7352,12 +8088,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -7367,7 +8103,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
; GFX908-NEXT: v_mov_b32_e32 v3, s10
-; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
@@ -7381,12 +8117,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX8-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -7396,7 +8132,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
; GFX8-NEXT: v_mov_b32_e32 v3, s10
-; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
@@ -7410,12 +8146,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX7-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, v0
@@ -7425,7 +8161,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v1
; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v5, v0
@@ -7439,12 +8175,12 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB18_1
+; GFX7-NEXT: s_cbranch_execnz .LBB22_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset:
+; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, v0
@@ -7454,7 +8190,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v1
; GFX6-NEXT: v_mov_b32_e32 v3, s10
-; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v5, v0
@@ -7469,16 +8205,16 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB18_1
+; GFX6-NEXT: s_cbranch_execnz .LBB22_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmax ptr addrspace(7) %gep, float %val seq_cst
+ %result = atomicrmw fmax ptr addrspace(7) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
-
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 118550169b6d6..2791162396a91 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -13,8 +13,8 @@
; float
; --------------------------------------------------------------------
-define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -28,7 +28,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -57,7 +57,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s4
@@ -68,7 +68,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -79,7 +79,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -107,7 +107,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -136,7 +136,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -165,7 +165,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
@@ -174,7 +174,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s8
@@ -184,12 +184,12 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -203,7 +203,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, s4
@@ -231,7 +231,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s4
@@ -242,7 +242,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -253,7 +253,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s8
@@ -280,7 +280,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s8
@@ -308,7 +308,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s8
@@ -336,7 +336,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
@@ -345,7 +345,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s8
@@ -355,12 +355,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr addrspace(7) %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -393,7 +393,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
@@ -454,7 +454,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX940-NEXT: v_mov_b32_e32 v0, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_mov_b32 s1, exec_lo
@@ -484,7 +484,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_mov_b32 s5, exec_lo
@@ -513,7 +513,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
@@ -572,7 +572,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
@@ -632,7 +632,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
@@ -692,7 +692,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[6:7], exec
@@ -718,7 +718,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b64 s[6:7], exec
@@ -745,144 +745,126 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall(ptr ad
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-; --------------------------------------------------------------------
-; double
-; --------------------------------------------------------------------
-
-define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset(ptr addrspace(7) inreg %ptr, double %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s4
-; GFX12-NEXT: s_addk_co_i32 s4, 0x800
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_mov_b32_e32 v6, s4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB3_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: v_mov_b32_e32 v1, s4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
-; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
-; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX940-NEXT: s_add_i32 s6, s4, 0x400
+; GFX940-NEXT: s_mov_b64 s[4:5], 0
+; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX940-NEXT: v_mov_b32_e32 v3, s6
+; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v5, v0
+; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
+; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX940-NEXT: s_cbranch_execnz .LBB3_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s4
-; GFX11-NEXT: s_addk_i32 s4, 0x800
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
-; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_min_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB3_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v0, s8
+; GFX90A-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX90A-NEXT: s_add_i32 s10, s8, 0x400
+; GFX90A-NEXT: s_mov_b64 s[8:9], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s10
+; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s8
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX908-NEXT: s_add_i32 s10, s8, 0x400
; GFX908-NEXT: s_mov_b64 s[8:9], 0
-; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s10
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v10, v1
-; GFX908-NEXT: v_mov_b32_e32 v9, v0
-; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v7
-; GFX908-NEXT: v_mov_b32_e32 v1, v8
-; GFX908-NEXT: v_mov_b32_e32 v2, v9
-; GFX908-NEXT: v_mov_b32_e32 v3, v10
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
@@ -890,32 +872,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX8-NEXT: s_add_i32 s10, s8, 0x400
; GFX8-NEXT: s_mov_b64 s[8:9], 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s10
; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v10, v1
-; GFX8-NEXT: v_mov_b32_e32 v9, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v7
-; GFX8-NEXT: v_mov_b32_e32 v1, v8
-; GFX8-NEXT: v_mov_b32_e32 v2, v9
-; GFX8-NEXT: v_mov_b32_e32 v3, v10
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
@@ -923,635 +901,1393 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX7-NEXT: v_mov_b32_e32 v1, s8
+; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX6-NEXT: v_mov_b32_e32 v1, s8
+; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret float %result
}
-define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset(ptr addrspace(7) inreg %ptr, double %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
-; GFX12-NEXT: s_addk_co_i32 s4, 0x800
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v6, s4
-; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB4_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: v_mov_b32_e32 v1, s4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
-; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
-; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX940-NEXT: s_add_i32 s6, s4, 0x400
+; GFX940-NEXT: s_mov_b64 s[4:5], 0
+; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX940-NEXT: v_mov_b32_e32 v3, s6
+; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v5, v0
+; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
+; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX940-NEXT: s_cbranch_execnz .LBB4_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX11-NEXT: s_addk_i32 s4, 0x800
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
-; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_min_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
-; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB4_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v0, s8
+; GFX90A-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX90A-NEXT: s_add_i32 s10, s8, 0x400
+; GFX90A-NEXT: s_mov_b64 s[8:9], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s10
+; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX908-NEXT: s_add_i32 s10, s8, 0x400
; GFX908-NEXT: s_mov_b64 s[8:9], 0
-; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s10
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, v3
-; GFX908-NEXT: v_mov_b32_e32 v9, v2
-; GFX908-NEXT: v_mov_b32_e32 v8, v1
-; GFX908-NEXT: v_mov_b32_e32 v7, v0
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
-; GFX908-NEXT: v_mov_b32_e32 v2, v7
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v3, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
-; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
-; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
+; GFX8-NEXT: s_add_i32 s10, s8, 0x400
; GFX8-NEXT: s_mov_b64 s[8:9], 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s10
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, v3
-; GFX8-NEXT: v_mov_b32_e32 v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v8, v1
-; GFX8-NEXT: v_mov_b32_e32 v7, v0
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[4:7], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v2, v7
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX7-NEXT: v_mov_b32_e32 v1, s8
+; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX6-NEXT: v_mov_b32_e32 v1, s8
+; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret float %result
}
-define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall(ptr addrspace(7) %ptr, double %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; --------------------------------------------------------------------
+; double
+; --------------------------------------------------------------------
+
+define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
-; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB5_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[5:6], v[5:6]
-; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB5_4 Depth 2
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[0:1], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
-; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
-; GFX12-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB5_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
+; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
-; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB5_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v7, v6
-; GFX940-NEXT: v_mov_b32_e32 v6, v5
-; GFX940-NEXT: s_mov_b64 s[2:3], exec
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX940-NEXT: v_readfirstlane_b32 s4, v0
-; GFX940-NEXT: v_readfirstlane_b32 s5, v1
-; GFX940-NEXT: v_readfirstlane_b32 s6, v2
-; GFX940-NEXT: v_readfirstlane_b32 s7, v3
-; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
-; GFX940-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX940-NEXT: ; implicit-def: $vgpr4
-; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB5_1
-; GFX940-NEXT: ; %bb.2:
-; GFX940-NEXT: s_mov_b64 exec, s[2:3]
+; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v0, v6
-; GFX940-NEXT: v_mov_b32_e32 v1, v7
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
-; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB5_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
-; GFX11-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB5_4 Depth 2
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
-; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
-; GFX11-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], 0 offen glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB5_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
-; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB5_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB5_1
-; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, v5
-; GFX10-NEXT: v_mov_b32_e32 v1, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v7, v6
-; GFX90A-NEXT: v_mov_b32_e32 v6, v5
-; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
-; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
-; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX90A-NEXT: ; implicit-def: $vgpr4
-; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
-; GFX90A-NEXT: ; %bb.2:
-; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v0, v6
-; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v8, v3
-; GFX908-NEXT: v_mov_b32_e32 v7, v2
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
+; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
-; GFX908-NEXT: v_add_u32_e32 v15, 0x800, v4
-; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_readfirstlane_b32 s8, v9
-; GFX908-NEXT: v_readfirstlane_b32 s9, v10
-; GFX908-NEXT: v_readfirstlane_b32 s10, v7
-; GFX908-NEXT: v_readfirstlane_b32 s11, v8
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
-; GFX908-NEXT: ; implicit-def: $vgpr4
-; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB5_1
-; GFX908-NEXT: ; %bb.2:
-; GFX908-NEXT: s_mov_b64 exec, s[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB5_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX908-NEXT: s_mov_b64 s[12:13], exec
-; GFX908-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v11
-; GFX908-NEXT: v_mov_b32_e32 v1, v12
-; GFX908-NEXT: v_mov_b32_e32 v2, v13
-; GFX908-NEXT: v_mov_b32_e32 v3, v14
-; GFX908-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT: v_readfirstlane_b32 s8, v9
-; GFX908-NEXT: v_readfirstlane_b32 s9, v10
-; GFX908-NEXT: v_readfirstlane_b32 s10, v7
-; GFX908-NEXT: v_readfirstlane_b32 s11, v8
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
-; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB5_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX908-NEXT: s_mov_b64 exec, s[12:13]
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v7
+; GFX908-NEXT: v_mov_b32_e32 v1, v8
+; GFX908-NEXT: v_mov_b32_e32 v2, v9
+; GFX908-NEXT: v_mov_b32_e32 v3, v10
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
-; GFX908-NEXT: v_mov_b32_e32 v14, v1
-; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v13, v0
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB5_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB5_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v8, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
-; GFX8-NEXT: v_add_u32_e32 v15, vcc, 0x800, v4
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_readfirstlane_b32 s8, v9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v10
-; GFX8-NEXT: v_readfirstlane_b32 s10, v7
-; GFX8-NEXT: v_readfirstlane_b32 s11, v8
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
-; GFX8-NEXT: ; implicit-def: $vgpr4
-; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB5_1
-; GFX8-NEXT: ; %bb.2:
-; GFX8-NEXT: s_mov_b64 exec, s[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: .LBB5_3: ; %atomicrmw.start
-; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB5_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX8-NEXT: s_mov_b64 s[12:13], exec
-; GFX8-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v11
-; GFX8-NEXT: v_mov_b32_e32 v1, v12
-; GFX8-NEXT: v_mov_b32_e32 v2, v13
-; GFX8-NEXT: v_mov_b32_e32 v3, v14
-; GFX8-NEXT: .LBB5_4: ; Parent Loop BB5_3 Depth=1
-; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX8-NEXT: v_readfirstlane_b32 s8, v9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v10
-; GFX8-NEXT: v_readfirstlane_b32 s10, v7
-; GFX8-NEXT: v_readfirstlane_b32 s11, v8
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
-; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
-; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
-; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB5_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB5_3 Depth=1
-; GFX8-NEXT: s_mov_b64 exec, s[12:13]
+; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
-; GFX8-NEXT: v_mov_b32_e32 v14, v1
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v13, v0
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB5_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB5_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_readfirstlane_b32 s8, v0
-; GFX7-NEXT: v_readfirstlane_b32 s9, v1
-; GFX7-NEXT: v_readfirstlane_b32 s10, v2
-; GFX7-NEXT: v_readfirstlane_b32 s11, v3
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX7-NEXT: ; implicit-def: $vgpr4
-; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB5_1
-; GFX7-NEXT: ; %bb.2:
-; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, v5
-; GFX7-NEXT: v_mov_b32_e32 v1, v6
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_readfirstlane_b32 s8, v0
-; GFX6-NEXT: v_readfirstlane_b32 s9, v1
-; GFX6-NEXT: v_readfirstlane_b32 s10, v2
-; GFX6-NEXT: v_readfirstlane_b32 s11, v3
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX6-NEXT: ; implicit-def: $vgpr4
-; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB5_1
-; GFX6-NEXT: ; %bb.2:
-; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v10, v3
+; GFX908-NEXT: v_mov_b32_e32 v9, v2
+; GFX908-NEXT: v_mov_b32_e32 v8, v1
+; GFX908-NEXT: v_mov_b32_e32 v7, v0
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v2, v7
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: v_mov_b32_e32 v3, v8
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB6_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v10, v3
+; GFX8-NEXT: v_mov_b32_e32 v9, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, v7
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: v_mov_b32_e32 v3, v8
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB6_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %unused = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
+; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_readfirstlane_b32 s4, v9
+; GFX12-NEXT: v_readfirstlane_b32 s5, v10
+; GFX12-NEXT: v_readfirstlane_b32 s6, v7
+; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
+; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-NEXT: ; %bb.2:
+; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[5:6], v[5:6]
+; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
+; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
+; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
+; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-NEXT: v_readfirstlane_b32 s4, v9
+; GFX12-NEXT: v_readfirstlane_b32 s5, v10
+; GFX12-NEXT: v_readfirstlane_b32 s6, v7
+; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB7_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
+; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_cbranch_execnz .LBB7_3
+; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v7, v6
+; GFX940-NEXT: v_mov_b32_e32 v6, v5
+; GFX940-NEXT: s_mov_b64 s[2:3], exec
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: v_readfirstlane_b32 s4, v0
+; GFX940-NEXT: v_readfirstlane_b32 s5, v1
+; GFX940-NEXT: v_readfirstlane_b32 s6, v2
+; GFX940-NEXT: v_readfirstlane_b32 s7, v3
+; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
+; GFX940-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX940-NEXT: ; implicit-def: $vgpr4
+; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB7_1
+; GFX940-NEXT: ; %bb.2:
+; GFX940-NEXT: s_mov_b64 exec, s[2:3]
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v6
+; GFX940-NEXT: v_mov_b32_e32 v1, v7
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
+; GFX11-NEXT: s_mov_b32 s1, 0
+; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-NEXT: v_readfirstlane_b32 s5, v10
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: v_readfirstlane_b32 s7, v8
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
+; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
+; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
+; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
+; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-NEXT: v_readfirstlane_b32 s5, v10
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: v_readfirstlane_b32 s7, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], 0 offen glc
+; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB7_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
+; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_cbranch_execnz .LBB7_3
+; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s8, v0
+; GFX10-NEXT: v_readfirstlane_b32 s9, v1
+; GFX10-NEXT: v_readfirstlane_b32 s10, v2
+; GFX10-NEXT: v_readfirstlane_b32 s11, v3
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
+; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
+; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB7_1
+; GFX10-NEXT: ; %bb.2:
+; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v7, v6
+; GFX90A-NEXT: v_mov_b32_e32 v6, v5
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
+; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
+; GFX90A-NEXT: ; %bb.2:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v0, v6
+; GFX90A-NEXT: v_mov_b32_e32 v1, v7
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v8, v3
+; GFX908-NEXT: v_mov_b32_e32 v7, v2
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_add_u32_e32 v15, 0x800, v4
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v9
+; GFX908-NEXT: v_readfirstlane_b32 s9, v10
+; GFX908-NEXT: v_readfirstlane_b32 s10, v7
+; GFX908-NEXT: v_readfirstlane_b32 s11, v8
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
+; GFX908-NEXT: ; implicit-def: $vgpr4
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB7_1
+; GFX908-NEXT: ; %bb.2:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Loop Header: Depth=1
+; GFX908-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
+; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v11
+; GFX908-NEXT: v_mov_b32_e32 v1, v12
+; GFX908-NEXT: v_mov_b32_e32 v2, v13
+; GFX908-NEXT: v_mov_b32_e32 v3, v14
+; GFX908-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX908-NEXT: v_readfirstlane_b32 s8, v9
+; GFX908-NEXT: v_readfirstlane_b32 s9, v10
+; GFX908-NEXT: v_readfirstlane_b32 s10, v7
+; GFX908-NEXT: v_readfirstlane_b32 s11, v8
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB7_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX908-NEXT: s_mov_b64 exec, s[12:13]
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
+; GFX908-NEXT: v_mov_b32_e32 v14, v1
+; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v13, v0
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_cbranch_execnz .LBB7_3
+; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v8, v3
+; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, 0x800, v4
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v9
+; GFX8-NEXT: v_readfirstlane_b32 s9, v10
+; GFX8-NEXT: v_readfirstlane_b32 s10, v7
+; GFX8-NEXT: v_readfirstlane_b32 s11, v8
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
+; GFX8-NEXT: ; implicit-def: $vgpr4
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_1
+; GFX8-NEXT: ; %bb.2:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: .LBB7_3: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Loop Header: Depth=1
+; GFX8-NEXT: ; Child Loop BB7_4 Depth 2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
+; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v11
+; GFX8-NEXT: v_mov_b32_e32 v1, v12
+; GFX8-NEXT: v_mov_b32_e32 v2, v13
+; GFX8-NEXT: v_mov_b32_e32 v3, v14
+; GFX8-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
+; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX8-NEXT: v_readfirstlane_b32 s8, v9
+; GFX8-NEXT: v_readfirstlane_b32 s9, v10
+; GFX8-NEXT: v_readfirstlane_b32 s10, v7
+; GFX8-NEXT: v_readfirstlane_b32 s11, v8
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
+; GFX8-NEXT: s_mov_b64 exec, s[12:13]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
+; GFX8-NEXT: v_mov_b32_e32 v14, v1
+; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v13, v0
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_3
+; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr4
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB7_1
+; GFX7-NEXT: ; %bb.2:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, v5
+; GFX7-NEXT: v_mov_b32_e32 v1, v6
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB7_1
+; GFX6-NEXT: ; %bb.2:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, v5
+; GFX6-NEXT: v_mov_b32_e32 v1, v6
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
+; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
+; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v7
+; GFX908-NEXT: v_mov_b32_e32 v1, v8
+; GFX908-NEXT: v_mov_b32_e32 v2, v9
+; GFX908-NEXT: v_mov_b32_e32 v3, v10
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB8_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB8_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
+ %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret double %result
+}
+
+define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: s_addk_co_i32 s4, 0x800
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v6, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
+; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_execnz .LBB9_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, s4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: s_addk_i32 s4, 0x800
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s8
+; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
+; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: s_add_i32 s10, s8, 0x800
+; GFX908-NEXT: s_mov_b64 s[8:9], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v7
+; GFX908-NEXT: v_mov_b32_e32 v1, v8
+; GFX908-NEXT: v_mov_b32_e32 v2, v9
+; GFX908-NEXT: v_mov_b32_e32 v3, v10
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_cbranch_execnz .LBB9_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[4:7], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX8-NEXT: s_add_i32 s10, s8, 0x800
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[4:7], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB9_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[4:7], 0 offen offset:2048 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v5
-; GFX6-NEXT: v_mov_b32_e32 v1, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret double %result
}
@@ -1559,8 +2295,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall(ptr a
; half
; --------------------------------------------------------------------
-define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) inreg %ptr, half %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1578,7 +2314,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -1600,13 +2336,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -1619,7 +2355,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX940-NEXT: s_not_b32 s7, s4
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX940-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v1
@@ -1636,13 +2372,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB6_1
+; GFX940-NEXT: s_cbranch_execnz .LBB10_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -1657,7 +2393,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -1680,13 +2416,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -1699,7 +2435,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v4, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v1
@@ -1718,13 +2454,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB6_1
+; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -1737,7 +2473,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX90A-NEXT: s_not_b32 s11, s8
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1753,13 +2489,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -1772,7 +2508,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX908-NEXT: s_not_b32 s11, s8
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1789,13 +2525,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB6_1
+; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -1808,7 +2544,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_max_f16_e32 v5, v0, v0
-; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1826,13 +2562,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB6_1
+; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -1846,7 +2582,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1865,14 +2601,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB6_1
+; GFX7-NEXT: s_cbranch_execnz .LBB10_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -1886,7 +2622,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -1906,7 +2642,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB6_1
+; GFX6-NEXT: s_cbranch_execnz .LBB10_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v2
@@ -1914,12 +2650,12 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset(ptr addrspace(7) i
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7) inreg %ptr, half %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1937,7 +2673,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -1959,12 +2695,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -1977,7 +2713,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX940-NEXT: s_not_b32 s7, s4
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX940-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v1
@@ -1994,12 +2730,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB7_1
+; GFX940-NEXT: s_cbranch_execnz .LBB11_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -2014,7 +2750,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -2037,12 +2773,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -2055,7 +2791,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v2, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v1
@@ -2074,12 +2810,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB7_1
+; GFX10-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -2092,7 +2828,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX90A-NEXT: s_not_b32 s11, s8
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2108,12 +2844,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -2126,7 +2862,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX908-NEXT: s_not_b32 s11, s8
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2143,12 +2879,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB7_1
+; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -2161,7 +2897,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2179,12 +2915,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB7_1
+; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -2198,7 +2934,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2217,12 +2953,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB7_1
+; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -2236,7 +2972,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -2256,18 +2992,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset(ptr addrspace(7)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB7_1
+; GFX6-NEXT: s_cbranch_execnz .LBB11_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr addrspace(7) %ptr, half %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, half %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2283,7 +3019,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -2296,14 +3032,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_max_num_f16_e32 v10, v5, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-NEXT: s_mov_b32 s2, exec_lo
@@ -2318,7 +3054,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX12-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -2333,8 +3069,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB8_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB12_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -2343,13 +3079,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB8_3
+; GFX12-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -2360,7 +3096,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: v_lshlrev_b32_e64 v4, v8, s0
; GFX940-NEXT: v_not_b32_e32 v10, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -2372,14 +3108,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX940-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB8_1
+; GFX940-NEXT: s_cbranch_execnz .LBB12_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
; GFX940-NEXT: v_max_f16_e32 v11, v5, v5
-; GFX940-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB12_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v4, v8, v7
; GFX940-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2389,7 +3125,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: s_mov_b64 s[8:9], exec
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX940-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -2403,8 +3139,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB8_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB12_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -2412,13 +3148,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB8_3
+; GFX940-NEXT: s_cbranch_execnz .LBB12_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -2431,7 +3167,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -2444,14 +3180,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-NEXT: s_mov_b32 s2, exec_lo
@@ -2466,7 +3202,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX11-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -2481,8 +3217,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB8_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB12_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -2492,13 +3228,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB8_3
+; GFX11-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -2509,7 +3245,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
-; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -2521,13 +3257,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB8_1
+; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
-; GFX10-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX10-NEXT: s_mov_b32 s6, exec_lo
@@ -2538,7 +3274,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -2552,8 +3288,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB8_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB12_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -2563,13 +3299,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB8_3
+; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -2580,7 +3316,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
; GFX90A-NEXT: v_not_b32_e32 v10, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -2592,14 +3328,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_max_f16_e32 v11, v5, v5
-; GFX90A-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB12_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v8, v7
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2608,7 +3344,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX90A-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -2621,8 +3357,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB8_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -2630,13 +3366,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB8_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB12_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -2647,7 +3383,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX908-NEXT: v_not_b32_e32 v9, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -2659,14 +3395,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB8_1
+; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_max_f16_e32 v10, v5, v5
-; GFX908-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB12_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX908-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2676,7 +3412,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX908-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -2689,8 +3425,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB8_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB12_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2698,13 +3434,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB8_3
+; GFX908-NEXT: s_cbranch_execnz .LBB12_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
@@ -2715,7 +3451,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX8-NEXT: v_not_b32_e32 v9, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -2727,14 +3463,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB8_1
+; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_max_f16_e32 v10, v5, v5
-; GFX8-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB12_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX8-NEXT: v_max_f16_e32 v4, v4, v4
@@ -2745,7 +3481,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX8-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -2758,8 +3494,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB8_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB12_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2767,13 +3503,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB8_3
+; GFX8-NEXT: s_cbranch_execnz .LBB12_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -2783,7 +3519,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -2794,15 +3530,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB8_1
+; GFX7-NEXT: s_cbranch_execnz .LBB12_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v4
-; GFX7-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB12_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
@@ -2814,7 +3550,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
; GFX7-NEXT: v_mov_b32_e32 v4, v5
; GFX7-NEXT: v_mov_b32_e32 v5, v6
-; GFX7-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX7-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -2827,8 +3563,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB8_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB12_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2836,14 +3572,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB8_3
+; GFX7-NEXT: s_cbranch_execnz .LBB12_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -2853,7 +3589,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -2864,15 +3600,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB8_1
+; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v4
-; GFX6-NEXT: .LBB8_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB8_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB12_4 Depth 2
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
@@ -2884,7 +3620,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
; GFX6-NEXT: v_mov_b32_e32 v4, v5
; GFX6-NEXT: v_mov_b32_e32 v5, v6
-; GFX6-NEXT: .LBB8_4: ; Parent Loop BB8_3 Depth=1
+; GFX6-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -2897,8 +3633,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB8_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB8_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB12_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -2906,7 +3642,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB8_3
+; GFX6-NEXT: s_cbranch_execnz .LBB12_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
@@ -2914,7 +3650,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
@@ -2922,8 +3658,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall(ptr add
; bfloat
; --------------------------------------------------------------------
-define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2941,7 +3677,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -2970,13 +3706,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB9_1
+; GFX12-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -2990,7 +3726,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX940-NEXT: s_movk_i32 s8, 0x7fff
-; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3012,13 +3748,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB9_1
+; GFX940-NEXT: s_cbranch_execnz .LBB13_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -3033,7 +3769,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -3063,13 +3799,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -3082,7 +3818,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v4, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3105,13 +3841,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB9_1
+; GFX10-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -3125,7 +3861,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX90A-NEXT: s_movk_i32 s12, 0x7fff
-; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3145,13 +3881,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -3165,7 +3901,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX908-NEXT: s_movk_i32 s12, 0x7fff
-; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3186,13 +3922,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB9_1
+; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -3205,7 +3941,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v0, s10
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -3229,13 +3965,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB9_1
+; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -3249,7 +3985,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3269,14 +4005,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB9_1
+; GFX7-NEXT: s_cbranch_execnz .LBB13_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -3290,7 +4026,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3311,7 +4047,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB9_1
+; GFX6-NEXT: s_cbranch_execnz .LBB13_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v2
@@ -3319,12 +4055,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3342,7 +4078,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_not_b32 s6, s5
; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -3371,12 +4107,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB10_1
+; GFX12-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_addk_i32 s4, 0x200
@@ -3390,7 +4126,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX940-NEXT: s_movk_i32 s8, 0x7fff
-; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3412,12 +4148,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB10_1
+; GFX940-NEXT: s_cbranch_execnz .LBB14_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s4, 0x200
@@ -3432,7 +4168,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
@@ -3462,12 +4198,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB10_1
+; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s8, 0x200
@@ -3480,7 +4216,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: s_not_b32 s10, s9
; GFX10-NEXT: buffer_load_dword v1, v2, s[4:7], 0 offen
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3503,12 +4239,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB10_1
+; GFX10-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s8, 0x200
@@ -3522,7 +4258,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s12, 0x7fff
-; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3542,12 +4278,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s8, 0x200
@@ -3561,7 +4297,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX908-NEXT: s_movk_i32 s12, 0x7fff
-; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -3582,12 +4318,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB10_1
+; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s8, 0x200
@@ -3600,7 +4336,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_not_b32 s11, s8
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v0, s10
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -3624,12 +4360,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB10_1
+; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s8, 0x200
@@ -3643,7 +4379,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_not_b32 s11, s8
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3663,12 +4399,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB10_1
+; GFX7-NEXT: s_cbranch_execnz .LBB14_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s8, 0x200
@@ -3682,7 +4418,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_not_b32 s11, s8
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -3703,18 +4439,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset(ptr addrspace(7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB10_1
+; GFX6-NEXT: s_cbranch_execnz .LBB14_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr addrspace(7) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3730,7 +4466,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -3743,14 +4479,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-NEXT: s_mov_b32 s2, exec_lo
@@ -3772,7 +4508,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX12-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -3787,8 +4523,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB11_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB15_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -3797,13 +4533,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB11_3
+; GFX12-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -3814,7 +4550,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: v_lshlrev_b32_e64 v4, v8, s0
; GFX940-NEXT: v_not_b32_e32 v10, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -3826,15 +4562,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX940-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB11_1
+; GFX940-NEXT: s_cbranch_execnz .LBB15_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v11, 16, v5
; GFX940-NEXT: s_movk_i32 s10, 0x7fff
-; GFX940-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB15_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX940-NEXT: s_mov_b64 s[8:9], exec
@@ -3849,7 +4585,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX940-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
-; GFX940-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX940-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -3863,8 +4599,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB11_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB15_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -3872,13 +4608,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB11_3
+; GFX940-NEXT: s_cbranch_execnz .LBB15_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -3891,7 +4627,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -3904,15 +4640,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB11_1
+; GFX11-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-NEXT: s_mov_b32 s2, exec_lo
@@ -3934,7 +4670,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX11-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -3949,8 +4685,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB11_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB15_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -3960,14 +4696,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB11_3
+; GFX11-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
@@ -3978,7 +4714,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
-; GFX10-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -3990,13 +4726,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB11_1
+; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX10-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_mov_b32 s6, exec_lo
@@ -4011,7 +4747,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -4025,8 +4761,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB11_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB15_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -4036,13 +4772,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB11_3
+; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -4053,7 +4789,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
; GFX90A-NEXT: v_not_b32_e32 v10, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -4065,15 +4801,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v11, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
-; GFX90A-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB15_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_min_f32_e32 v4, v4, v11
@@ -4086,7 +4822,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX90A-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -4099,8 +4835,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB11_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -4108,13 +4844,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB11_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
@@ -4125,7 +4861,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX908-NEXT: v_not_b32_e32 v9, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -4137,15 +4873,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB11_1
+; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
-; GFX908-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB15_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_min_f32_e32 v4, v4, v10
@@ -4159,7 +4895,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX908-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -4172,8 +4908,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB11_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB15_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4181,13 +4917,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB11_3
+; GFX908-NEXT: s_cbranch_execnz .LBB15_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
@@ -4198,7 +4934,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
; GFX8-NEXT: v_not_b32_e32 v9, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -4210,14 +4946,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB11_1
+; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX8-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB15_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_min_f32_e32 v4, v4, v10
@@ -4233,7 +4969,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX8-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -4246,8 +4982,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB11_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB15_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4255,13 +4991,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB11_3
+; GFX8-NEXT: s_cbranch_execnz .LBB15_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -4271,7 +5007,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -4282,15 +5018,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB11_1
+; GFX7-NEXT: s_cbranch_execnz .LBB15_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
-; GFX7-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB15_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
@@ -4303,7 +5039,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: v_mov_b32_e32 v4, v5
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v5, v6
-; GFX7-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX7-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -4316,8 +5052,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB11_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB15_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4325,14 +5061,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB11_3
+; GFX7-NEXT: s_cbranch_execnz .LBB15_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
@@ -4342,7 +5078,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -4353,15 +5089,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB11_1
+; GFX6-NEXT: s_cbranch_execnz .LBB15_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
-; GFX6-NEXT: .LBB11_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB11_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB15_4 Depth 2
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
@@ -4374,7 +5110,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: v_mov_b32_e32 v4, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v5, v6
-; GFX6-NEXT: .LBB11_4: ; Parent Loop BB11_3 Depth=1
+; GFX6-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -4387,8 +5123,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB11_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB11_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB15_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -4396,7 +5132,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB11_3
+; GFX6-NEXT: s_cbranch_execnz .LBB15_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
@@ -4404,7 +5140,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
@@ -4412,8 +5148,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall(ptr
; <2 x half>
; --------------------------------------------------------------------
-define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4427,7 +5163,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v5, v0
@@ -4444,12 +5180,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -4459,7 +5195,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_pk_max_f16 v2, v1, v1
; GFX940-NEXT: v_mov_b32_e32 v3, s6
-; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v0
@@ -4474,12 +5210,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB12_1
+; GFX940-NEXT: s_cbranch_execnz .LBB16_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
@@ -4489,7 +5225,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v5, v0
@@ -4507,12 +5243,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB12_1
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, v0
@@ -4522,7 +5258,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -4538,12 +5274,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB12_1
+; GFX10-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -4553,7 +5289,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: v_mov_b32_e32 v3, s10
-; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
@@ -4566,12 +5302,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -4581,7 +5317,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_pk_max_f16 v2, v1, v1
; GFX908-NEXT: v_mov_b32_e32 v3, s10
-; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
@@ -4595,12 +5331,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB12_1
+; GFX908-NEXT: s_cbranch_execnz .LBB16_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -4611,7 +5347,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v3, v1, v1
; GFX8-NEXT: v_mov_b32_e32 v4, s10
-; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -4628,12 +5364,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB12_1
+; GFX8-NEXT: s_cbranch_execnz .LBB16_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -4649,7 +5385,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_mov_b32_e32 v4, s10
-; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -4674,12 +5410,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB12_1
+; GFX7-NEXT: s_cbranch_execnz .LBB16_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -4695,7 +5431,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX6-NEXT: v_mov_b32_e32 v4, s10
-; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -4721,18 +5457,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset(ptr addrsp
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB12_1
+; GFX6-NEXT: s_cbranch_execnz .LBB16_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4745,7 +5481,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX12-NEXT: v_mov_b32_e32 v3, s4
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v0, v1, v1
@@ -4761,12 +5497,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, s4
@@ -4775,7 +5511,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_pk_max_f16 v2, v0, v0
; GFX940-NEXT: v_mov_b32_e32 v3, s6
-; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4790,12 +5526,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: v_mov_b32_e32 v1, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB13_1
+; GFX940-NEXT: s_cbranch_execnz .LBB17_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s4
@@ -4804,7 +5540,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX11-NEXT: v_mov_b32_e32 v3, s4
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4821,12 +5557,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -4835,7 +5571,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, s8
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: buffer_load_dword v1, v1, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4851,12 +5587,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB13_1
+; GFX10-NEXT: s_cbranch_execnz .LBB17_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s8
@@ -4865,7 +5601,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s10
-; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4878,12 +5614,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s8
@@ -4892,7 +5628,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s10
-; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
@@ -4906,12 +5642,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB13_1
+; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s8
@@ -4921,7 +5657,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s10
-; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -4938,12 +5674,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB13_1
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -4959,7 +5695,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v5
; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
@@ -4984,12 +5720,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB13_1
+; GFX7-NEXT: s_cbranch_execnz .LBB17_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -5005,7 +5741,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v5
; GFX6-NEXT: v_mov_b32_e32 v2, s10
-; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
@@ -5031,18 +5767,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset(ptr addrspace(
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB13_1
+; GFX6-NEXT: s_cbranch_execnz .LBB17_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5051,7 +5787,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -5065,14 +5801,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v4, v6, v6
; GFX12-NEXT: s_mov_b32 s2, exec_lo
@@ -5081,7 +5817,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: v_pk_min_num_f16 v5, v4, v8
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -5096,8 +5832,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB18_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -5106,18 +5842,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB14_3
+; GFX12-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -5130,21 +5866,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX940-NEXT: ; implicit-def: $vgpr4
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB14_1
+; GFX940-NEXT: s_cbranch_execnz .LBB18_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
; GFX940-NEXT: v_pk_max_f16 v9, v5, v5
-; GFX940-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB18_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v4, v7, v7
; GFX940-NEXT: s_mov_b64 s[8:9], exec
; GFX940-NEXT: v_pk_min_f16 v6, v4, v9
; GFX940-NEXT: buffer_wbl2 sc1
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
-; GFX940-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX940-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -5158,8 +5894,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB14_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB18_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -5167,19 +5903,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB14_3
+; GFX940-NEXT: s_cbranch_execnz .LBB18_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -5193,14 +5929,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: buffer_load_b32 v6, v4, s[4:7], 0 offen offset:1024
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v4, v6, v6
; GFX11-NEXT: s_mov_b32 s2, exec_lo
@@ -5209,7 +5945,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: v_pk_min_f16 v5, v4, v8
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -5224,8 +5960,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB18_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -5235,19 +5971,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB14_3
+; GFX11-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -5260,13 +5996,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: ; implicit-def: $vgpr4
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_1
+; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX10-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v4, v6, v6
; GFX10-NEXT: s_mov_b32 s6, exec_lo
@@ -5274,7 +6010,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_pk_min_f16 v5, v4, v8
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -5288,8 +6024,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB18_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -5299,18 +6035,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB14_3
+; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -5323,20 +6059,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_pk_max_f16 v9, v5, v5
-; GFX90A-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB18_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v4, v7, v7
; GFX90A-NEXT: v_pk_min_f16 v6, v4, v9
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX90A-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -5349,8 +6085,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB14_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -5358,18 +6094,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB14_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -5382,21 +6118,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: ; implicit-def: $vgpr4
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB14_1
+; GFX908-NEXT: s_cbranch_execnz .LBB18_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX908-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB18_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v4, v6, v6
; GFX908-NEXT: v_pk_min_f16 v5, v4, v8
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX908-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -5409,8 +6145,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB14_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB18_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -5418,18 +6154,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB14_3
+; GFX908-NEXT: s_cbranch_execnz .LBB18_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -5442,15 +6178,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: ; implicit-def: $vgpr4
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB14_1
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
-; GFX8-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB18_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v4, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
@@ -5460,7 +6196,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX8-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -5473,8 +6209,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB14_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB18_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -5482,18 +6218,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB14_3
+; GFX8-NEXT: s_cbranch_execnz .LBB18_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -5505,7 +6241,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_1
+; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -5517,9 +6253,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v11, v8
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB18_4 Depth 2
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[12:13], exec
@@ -5535,7 +6271,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
; GFX7-NEXT: v_mov_b32_e32 v8, v6
; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -5548,8 +6284,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB18_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
@@ -5559,19 +6295,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_3
+; GFX7-NEXT: s_cbranch_execnz .LBB18_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: v_mov_b32_e32 v1, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -5583,7 +6319,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_1
+; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -5595,9 +6331,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v11, v8
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: .LBB14_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB14_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB18_4 Depth 2
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
@@ -5614,7 +6350,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
; GFX6-NEXT: v_mov_b32_e32 v8, v6
; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: .LBB14_4: ; Parent Loop BB14_3 Depth=1
+; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -5627,8 +6363,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB14_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB18_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
@@ -5638,7 +6374,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_3
+; GFX6-NEXT: s_cbranch_execnz .LBB18_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v4
@@ -5646,7 +6382,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
@@ -5654,8 +6390,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; <2 x bfloat>
; --------------------------------------------------------------------
-define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5670,7 +6406,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
; GFX12-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v0
@@ -5703,12 +6439,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-NEXT: s_cbranch_execnz .LBB19_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -5721,7 +6457,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX940-NEXT: s_mov_b32 s9, 0x7060302
; GFX940-NEXT: v_mov_b32_e32 v4, s4
-; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v0
@@ -5748,12 +6484,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX940-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX940-NEXT: s_cbranch_execnz .LBB15_1
+; GFX940-NEXT: s_cbranch_execnz .LBB19_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
@@ -5766,7 +6502,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v0
@@ -5800,13 +6536,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-NEXT: s_cbranch_execnz .LBB19_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, v0
@@ -5817,7 +6553,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -5846,12 +6582,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB15_1
+; GFX10-NEXT: s_cbranch_execnz .LBB19_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -5864,7 +6600,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s13, 0x7060302
; GFX90A-NEXT: v_mov_b32_e32 v4, s8
-; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
@@ -5890,12 +6626,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -5908,7 +6644,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s13, 0x7060302
; GFX908-NEXT: v_mov_b32_e32 v4, s8
-; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
@@ -5935,12 +6671,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX908-NEXT: s_cbranch_execnz .LBB15_1
+; GFX908-NEXT: s_cbranch_execnz .LBB19_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -5951,7 +6687,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -5981,12 +6717,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_cbranch_execnz .LBB15_1
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -6001,7 +6737,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v4, s8
-; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -6023,12 +6759,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX7-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX7-NEXT: s_cbranch_execnz .LBB15_1
+; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -6043,7 +6779,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v4, s8
-; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -6066,18 +6802,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset(ptr add
; GFX6-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX6-NEXT: s_cbranch_execnz .LBB15_1
+; GFX6-NEXT: s_cbranch_execnz .LBB19_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6090,7 +6826,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v0
; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
@@ -6121,12 +6857,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: s_cbranch_execnz .LBB20_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, s4
@@ -6138,7 +6874,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX940-NEXT: s_mov_b32 s9, 0x7060302
; GFX940-NEXT: v_mov_b32_e32 v4, s4
-; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6165,12 +6901,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX940-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX940-NEXT: v_mov_b32_e32 v1, v6
; GFX940-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX940-NEXT: s_cbranch_execnz .LBB16_1
+; GFX940-NEXT: s_cbranch_execnz .LBB20_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_lshlrev_b32 v2, 16, v0
@@ -6181,7 +6917,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
@@ -6213,13 +6949,13 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: s_cbranch_execnz .LBB20_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s8
@@ -6229,7 +6965,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v4, s8
; GFX10-NEXT: buffer_load_dword v1, v1, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_mov_b32 s9, 0
-; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6258,12 +6994,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_cbranch_execnz .LBB16_1
+; GFX10-NEXT: s_cbranch_execnz .LBB20_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s8
@@ -6275,7 +7011,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s13, 0x7060302
; GFX90A-NEXT: v_mov_b32_e32 v4, s8
-; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6301,12 +7037,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX90A-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s8
@@ -6318,7 +7054,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b32 s13, 0x7060302
; GFX908-NEXT: v_mov_b32_e32 v4, s8
-; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6345,12 +7081,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX908-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s8
@@ -6360,7 +7096,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -6390,12 +7126,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s8
@@ -6410,7 +7146,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -6432,12 +7168,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX7-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX7-NEXT: s_cbranch_execnz .LBB16_1
+; GFX7-NEXT: s_cbranch_execnz .LBB20_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s8
@@ -6452,7 +7188,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -6475,18 +7211,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GFX6-NEXT: s_cbranch_execnz .LBB16_1
+; GFX6-NEXT: s_cbranch_execnz .LBB20_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
- %unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall(ptr addrspace(7) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6495,7 +7231,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-NEXT: v_readfirstlane_b32 s6, v2
@@ -6509,15 +7245,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-NEXT: buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s1
; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX12-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX12-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v6
@@ -6541,7 +7277,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
; GFX12-NEXT: v_mov_b32_e32 v4, v5
; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX12-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-NEXT: v_readfirstlane_b32 s5, v1
@@ -6556,8 +7292,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_4
+; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -6566,18 +7302,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB17_3
+; GFX12-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX940-NEXT: s_mov_b64 s[2:3], exec
-; GFX940-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
; GFX940-NEXT: v_readfirstlane_b32 s6, v2
@@ -6590,7 +7326,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX940-NEXT: ; implicit-def: $vgpr4
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB17_1
+; GFX940-NEXT: s_cbranch_execnz .LBB21_1
; GFX940-NEXT: ; %bb.2:
; GFX940-NEXT: s_mov_b64 exec, s[2:3]
; GFX940-NEXT: s_mov_b64 s[2:3], 0
@@ -6598,9 +7334,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX940-NEXT: s_movk_i32 s10, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
; GFX940-NEXT: s_mov_b32 s11, 0x7060302
-; GFX940-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX940-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Loop Header: Depth=1
-; GFX940-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX940-NEXT: ; Child Loop BB21_4 Depth 2
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX940-NEXT: v_min_f32_e32 v4, v4, v9
@@ -6621,7 +7357,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v11, vcc
; GFX940-NEXT: v_perm_b32 v6, v5, v4, s11
; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
-; GFX940-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX940-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
; GFX940-NEXT: v_readfirstlane_b32 s4, v0
; GFX940-NEXT: v_readfirstlane_b32 s5, v1
@@ -6635,8 +7371,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB17_4
-; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX940-NEXT: s_cbranch_execnz .LBB21_4
+; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX940-NEXT: s_mov_b64 exec, s[8:9]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -6644,19 +7380,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX940-NEXT: v_mov_b32_e32 v7, v4
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB17_3
+; GFX940-NEXT: s_cbranch_execnz .LBB21_3
; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v4
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: v_readfirstlane_b32 s6, v2
@@ -6670,16 +7406,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-NEXT: buffer_load_b32 v6, v4, s[4:7], 0 offen offset:1024
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX11-NEXT: ; Child Loop BB21_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
@@ -6703,7 +7439,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
; GFX11-NEXT: v_mov_b32_e32 v4, v5
; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
@@ -6718,8 +7454,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], 0 offen glc
; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_4
+; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -6729,20 +7465,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB17_3
+; GFX11-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
; GFX10-NEXT: v_readfirstlane_b32 s10, v2
@@ -6755,14 +7491,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: ; implicit-def: $vgpr4
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB17_1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
@@ -6783,7 +7519,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
; GFX10-NEXT: v_mov_b32_e32 v4, v5
; GFX10-NEXT: v_mov_b32_e32 v5, v6
-; GFX10-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_readfirstlane_b32 s9, v1
@@ -6797,8 +7533,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB17_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_4
+; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX10-NEXT: s_mov_b32 exec_lo, s6
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
@@ -6808,18 +7544,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_waitcnt_depctr 0xffe3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB17_3
+; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
-; GFX90A-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
@@ -6832,7 +7568,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
@@ -6840,9 +7576,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
-; GFX90A-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX90A-NEXT: ; Child Loop BB21_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX90A-NEXT: v_min_f32_e32 v4, v4, v9
@@ -6861,7 +7597,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_perm_b32 v6, v5, v4, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX90A-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -6874,8 +7610,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB17_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_4
+; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
@@ -6883,18 +7619,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB17_3
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
-; GFX908-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
; GFX908-NEXT: v_readfirstlane_b32 s10, v2
@@ -6907,7 +7643,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: ; implicit-def: $vgpr4
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
@@ -6915,9 +7651,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
-; GFX908-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX908-NEXT: ; Child Loop BB21_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX908-NEXT: v_min_f32_e32 v4, v4, v8
@@ -6937,7 +7673,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v4, v5
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v5, v6
-; GFX908-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -6950,8 +7686,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_4
+; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -6959,18 +7695,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_3
+; GFX908-NEXT: s_cbranch_execnz .LBB21_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v2
@@ -6983,15 +7719,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: ; implicit-def: $vgpr4
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX8-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX8-NEXT: ; Child Loop BB21_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX8-NEXT: v_min_f32_e32 v4, v4, v8
@@ -7014,7 +7750,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v4, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v5, v6
-; GFX8-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -7027,8 +7763,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_4
+; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
@@ -7036,18 +7772,18 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_3
+; GFX8-NEXT: s_cbranch_execnz .LBB21_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
-; GFX7-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
; GFX7-NEXT: v_readfirstlane_b32 s10, v2
@@ -7059,7 +7795,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
@@ -7070,9 +7806,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX7-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB21_4 Depth 2
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v7
@@ -7086,7 +7822,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_mov_b32_e32 v7, v5
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -7099,8 +7835,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB17_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_4
+; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
@@ -7109,19 +7845,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB17_3
+; GFX7-NEXT: s_cbranch_execnz .LBB21_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall:
+; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
-; GFX6-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
; GFX6-NEXT: v_readfirstlane_b32 s10, v2
@@ -7133,7 +7869,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB17_1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
@@ -7144,9 +7880,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX6-NEXT: .LBB17_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB17_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB21_4 Depth 2
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v7
@@ -7160,7 +7896,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB17_4: ; Parent Loop BB17_3 Depth=1
+; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -7173,8 +7909,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB17_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB17_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_4
+; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
@@ -7184,14 +7920,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB17_3
+; GFX6-NEXT: s_cbranch_execnz .LBB21_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
@@ -7199,8 +7935,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; misc
; --------------------------------------------------------------------
-define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
+; GFX12-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7213,7 +7949,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_max_num_f32 v2, v1, v1
; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v5, v0
@@ -7230,12 +7966,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: s_cbranch_execnz .LBB22_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX940-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v1, v0
@@ -7245,7 +7981,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: s_mov_b64 s[4:5], 0
; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
; GFX940-NEXT: v_mov_b32_e32 v3, s6
-; GFX940-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v0
@@ -7259,12 +7995,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX940-NEXT: s_cbranch_execnz .LBB18_1
+; GFX940-NEXT: s_cbranch_execnz .LBB22_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX11-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
@@ -7273,7 +8009,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_max_f32 v2, v1, v1
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v5, v0
@@ -7291,12 +8027,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: s_cbranch_execnz .LBB22_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, v0
@@ -7306,7 +8042,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: v_max_f32_e32 v2, v1, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -7322,12 +8058,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB18_1
+; GFX10-NEXT: s_cbranch_execnz .LBB22_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
@@ -7337,7 +8073,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: s_mov_b64 s[8:9], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
; GFX90A-NEXT: v_mov_b32_e32 v3, s10
-; GFX90A-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
@@ -7352,12 +8088,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, v0
@@ -7367,7 +8103,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: s_mov_b64 s[8:9], 0
; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
; GFX908-NEXT: v_mov_b32_e32 v3, s10
-; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
@@ -7381,12 +8117,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX8-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, v0
@@ -7396,7 +8132,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
; GFX8-NEXT: v_mov_b32_e32 v3, s10
-; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
@@ -7410,12 +8146,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX7-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, v0
@@ -7425,7 +8161,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v1
; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v5, v0
@@ -7439,12 +8175,12 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB18_1
+; GFX7-NEXT: s_cbranch_execnz .LBB22_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset:
+; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, v0
@@ -7454,7 +8190,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v1
; GFX6-NEXT: v_mov_b32_e32 v3, s10
-; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v5, v0
@@ -7469,16 +8205,16 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB18_1
+; GFX6-NEXT: s_cbranch_execnz .LBB22_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
- %result = atomicrmw fmin ptr addrspace(7) %gep, float %val seq_cst
+ %result = atomicrmw fmin ptr addrspace(7) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
-
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index a05c05f0b5a63..e6d6652c8b7b7 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -12,8 +12,8 @@
; float
; --------------------------------------------------------------------
-define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32:
+define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -26,7 +26,7 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -51,7 +51,7 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -61,7 +61,7 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -71,7 +71,7 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -95,7 +95,7 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -119,7 +119,7 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -143,19 +143,19 @@ define float @flat_agent_atomic_fmax_ret_f32(ptr %ptr, float %val) #0 {
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -168,7 +168,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -193,7 +193,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -203,7 +203,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -215,7 +215,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -239,7 +239,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -263,7 +263,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -288,7 +288,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -298,12 +298,12 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -316,7 +316,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v4, v0
@@ -348,7 +348,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -360,7 +360,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -372,7 +372,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -399,7 +399,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -426,7 +426,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -451,7 +451,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -461,12 +461,12 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 -512
- %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32:
+define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -479,7 +479,7 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -503,7 +503,7 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -514,7 +514,7 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -525,7 +525,7 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -548,7 +548,7 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -571,7 +571,7 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -594,19 +594,19 @@ define void @flat_agent_atomic_fmax_noret_f32(ptr %ptr, float %val) #0 {
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_atomic_fmax v[0:1], v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -619,7 +619,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -643,7 +643,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -654,7 +654,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -667,7 +667,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -690,7 +690,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -713,7 +713,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -738,7 +738,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -748,12 +748,12 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -766,7 +766,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -796,7 +796,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -809,7 +809,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -822,7 +822,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -849,7 +849,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -876,7 +876,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -901,7 +901,7 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -911,12 +911,12 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 -512
- %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -947,7 +947,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -972,7 +972,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -1000,7 +1000,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -1027,7 +1027,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1053,7 +1053,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1077,7 +1077,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -1102,7 +1102,7 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fc, v0
@@ -1127,12 +1127,12 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, float %val seq_cst
+ %result = atomicrmw fmax ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1162,7 +1162,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1186,7 +1186,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -1213,7 +1213,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -1240,7 +1240,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1265,7 +1265,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1288,7 +1288,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -1313,7 +1313,7 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -1338,16 +1338,12 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, float %val seq_cst
+ %unused = atomicrmw fmax ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-; --------------------------------------------------------------------
-; float with ftz/daz
-; --------------------------------------------------------------------
-
-define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1360,7 +1356,7 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -1385,7 +1381,7 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1395,7 +1391,7 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1405,7 +1401,7 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -1429,7 +1425,7 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -1453,7 +1449,7 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -1477,19 +1473,19 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1497,15 +1493,15 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_max_num_f32 v0, v[0:1], v2 offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_max_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1515,7 +1511,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1527,21 +1523,19 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_max_f32 v0, v[0:1], v2 offset:2044 glc
+; GFX11-NEXT: flat_atomic_max_f32 v0, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1549,10 +1543,10 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1561,7 +1555,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1573,10 +1567,10 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1585,7 +1579,7 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -1597,47 +1591,47 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; --------------------------------------------------------------------
+; float with ftz/daz
+; --------------------------------------------------------------------
+
+define float @flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1645,48 +1639,39 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_max_num_f32 v0, v[0:1], v2 offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_max_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v4, v0
-; GFX940-NEXT: v_mov_b32_e32 v5, v1
-; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v4
-; GFX940-NEXT: s_movk_i32 s0, 0xf800
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v5, vcc
-; GFX940-NEXT: flat_load_dword v0, v[0:1]
-; GFX940-NEXT: s_mov_b32 s1, -1
-; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[0:1]
+; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v1, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v3, v0
-; GFX940-NEXT: v_max_f32_e32 v0, v3, v3
-; GFX940-NEXT: v_max_f32_e32 v2, v0, v1
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB10_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_max_f32 v0, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1694,11 +1679,9 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1706,101 +1689,91 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_load_dword v0, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v0, v1
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: flat_load_dword v0, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v6, v0
-; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
-; GFX908-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 -512
- %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @flat_agent_atomic_fmax_noret_f32__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1808,139 +1781,147 @@ define void @flat_agent_atomic_fmax_noret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_max_num_f32 v[0:1], v2
-; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: flat_atomic_max_num_f32 v0, v[0:1], v2 offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_load_dword v3, v[0:1]
+; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB11_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_max_f32 v[0:1], v2
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_max_f32 v0, v[0:1], v2 offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmax v[0:1], v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_atomic_fmax v[0:1], v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr %ptr, i64 511
+ %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1948,146 +1929,162 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr %ptr, floa
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_max_num_f32 v[0:1], v2 offset:2044
-; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: flat_atomic_max_num_f32 v0, v[0:1], v2 offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX940-NEXT: v_mov_b32_e32 v4, v0
+; GFX940-NEXT: v_mov_b32_e32 v5, v1
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v4
+; GFX940-NEXT: s_movk_i32 s0, 0xf800
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v5, vcc
+; GFX940-NEXT: flat_load_dword v0, v[0:1]
+; GFX940-NEXT: s_mov_b32 s1, -1
+; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v1, v2, v2
; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v3, v0
+; GFX940-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX940-NEXT: v_max_f32_e32 v2, v0, v1
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
+; GFX940-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB12_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_max_f32 v[0:1], v2 offset:2044
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_max_f32 v0, v[0:1], v2 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmax v[0:1], v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_load_dword v0, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v0, v1
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
+; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX908-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_atomic_fmax v[0:1], v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX7-NEXT: flat_atomic_fmax v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr %ptr, i64 -512
+ %result = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+define void @flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2095,21 +2092,15 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_max_num_f32 v[0:1], v2 offset:-2048
+; GFX12-NEXT: flat_atomic_max_num_f32 v[0:1], v2
; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX940-NEXT: s_movk_i32 s0, 0xf800
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX940-NEXT: flat_load_dword v3, v[4:5]
-; GFX940-NEXT: s_mov_b32 s1, -1
-; GFX940-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2130,11 +2121,9 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_max_f32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -2143,11 +2132,9 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmax v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -2156,65 +2143,55 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_load_dword v1, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: flat_load_dword v1, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
@@ -2235,238 +2212,329 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmax v[0:1], v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 -512
- %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-NEXT: flat_atomic_max_num_f32 v[0:1], v2 offset:2044
+; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v5, v3
-; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: buffer_inv sc0 sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB14_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_atomic_max_f32 v[0:1], v2 offset:2044
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v6, v0
-; GFX10-NEXT: v_max_f32_e32 v0, v6, v6
-; GFX10-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax v[0:1], v2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_1
-; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fc, v0
-; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v0, v[3:4]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX7-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX7-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_atomic_fmax v[0:1], v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, float %val seq_cst
- ret float %result
+ %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_max_num_f32 v[0:1], v2 offset:-2048
+; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_movk_i32 s0, 0xf800
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_load_dword v3, v[4:5]
+; GFX940-NEXT: s_mov_b32 s1, -1
+; GFX940-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB15_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_max_f32 v[0:1], v2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax v[0:1], v2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_load_dword v1, v[0:1]
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
+; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT: flat_load_dword v1, v[0:1]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB15_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB15_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX7-NEXT: flat_atomic_fmax v[0:1], v2
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr %ptr, i64 -512
+ %unused = atomicrmw fmax ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2474,224 +2542,751 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr %ptr, flo
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
+; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc0 sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB15_1
+; GFX940-NEXT: s_cbranch_execnz .LBB16_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f32_e32 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX10-NEXT: flat_load_dword v0, v[3:4]
+; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX10-NEXT: v_max_f32_e32 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB15_1
+; GFX10-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_load_dword v0, v[3:4]
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX7-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX7-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB16_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr %ptr, i64 511
+ %result = atomicrmw fmax ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
+}
+
+define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc0 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB17_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB17_1
+; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_load_dword v3, v[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v3, v2
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr %ptr, i64 511
+ %unused = atomicrmw fmax ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+; --------------------------------------------------------------------
+; double
+; --------------------------------------------------------------------
+
+define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] offset:2040 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] offset:2040 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
-; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB15_1
+; GFX908-NEXT: s_cbranch_execnz .LBB19_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[0:1]
+; GFX8-NEXT: flat_load_dword v0, v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB15_1
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v3, v2
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB15_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, float %val seq_cst
- ret void
+ %gep = getelementptr double, ptr %ptr, i64 255
+ %result = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
}
-; --------------------------------------------------------------------
-; double
-; --------------------------------------------------------------------
-
-define double @flat_agent_atomic_fmax_ret_f64(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64:
+define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
@@ -2699,59 +3294,69 @@ define double @flat_agent_atomic_fmax_ret_f64(ptr %ptr, double %val) #0 {
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: s_cbranch_execnz .LBB20_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v0
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v5
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v4, vcc_lo
+; GFX11-NEXT: v_add_co_u32 v5, vcc_lo, 0xfffff800, v5
+; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, -1, v4, vcc_lo
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[5:6], v[7:10] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: s_cbranch_execnz .LBB20_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2759,429 +3364,411 @@ define double @flat_agent_atomic_fmax_ret_f64(ptr %ptr, double %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_add_co_u32_e32 v5, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v1
+; GFX908-NEXT: s_mov_b64 s[4:5], vcc
+; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v4, vcc
+; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: v_addc_co_u32_e64 v6, vcc, -1, v4, s[4:5]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[5:6], v[7:10] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v1, v[0:1]
+; GFX8-NEXT: flat_load_dword v0, v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr %ptr, i64 -256
+ %result = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret double %result
}
-define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] offset:2040 sc0
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] offset:2040 glc
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:2040 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v1, v[0:1]
-; GFX8-NEXT: flat_load_dword v0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 255
- %result = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %unused = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: s_cbranch_execnz .LBB22_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX940-NEXT: s_nop 1
-; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3] offset:2040
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v0
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v5
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v4, vcc_lo
-; GFX11-NEXT: v_add_co_u32 v5, vcc_lo, 0xfffff800, v5
-; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, -1, v4, vcc_lo
-; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1]
-; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[5:6], v[7:10] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: s_cbranch_execnz .LBB22_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3] offset:2040
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v5, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_mov_b32_e32 v4, v1
-; GFX908-NEXT: s_mov_b64 s[4:5], vcc
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v4, vcc
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_addc_co_u32_e64 v6, vcc, -1, v4, s[4:5]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v10, v1
-; GFX908-NEXT: v_mov_b32_e32 v9, v0
-; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[5:6], v[7:10] glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v1, v[0:1]
-; GFX8-NEXT: flat_load_dword v0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v5, v[0:1]
+; GFX8-NEXT: flat_load_dword v4, v[6:7]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 -256
- %result = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %gep = getelementptr double, ptr %ptr, i64 255
+ %unused = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @flat_agent_atomic_fmax_noret_f64(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f64:
+define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
@@ -3189,27 +3776,34 @@ define void @flat_agent_atomic_fmax_noret_f64(ptr %ptr, double %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: s_cbranch_execnz .LBB23_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: flat_load_b64 v[4:5], v[4:5]
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
@@ -3225,14 +3819,16 @@ define void @flat_agent_atomic_fmax_noret_f64(ptr %ptr, double %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: s_cbranch_execnz .LBB23_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -3241,53 +3837,62 @@ define void @flat_agent_atomic_fmax_noret_f64(ptr %ptr, double %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffff800, v0
+; GFX908-NEXT: s_mov_b64 s[4:5], vcc
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
+; GFX908-NEXT: v_addc_co_u32_e64 v7, vcc, -1, v1, s[4:5]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[8:9]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB19_1
+; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: flat_load_dword v5, v[5:6]
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, -1, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v5, v[0:1]
+; GFX8-NEXT: flat_load_dword v4, v[6:7]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
@@ -3295,352 +3900,337 @@ define void @flat_agent_atomic_fmax_noret_f64(ptr %ptr, double %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr %ptr, i64 -256
+ %unused = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB20_1
+; GFX12-NEXT: s_cbranch_execnz .LBB24_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3] offset:2040
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB20_1
+; GFX11-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3] offset:2040
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] offset:2040 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB20_1
+; GFX908-NEXT: s_cbranch_execnz .LBB24_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[0:1]
-; GFX8-NEXT: flat_load_dword v4, v[6:7]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[0:1]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB20_1
+; GFX8-NEXT: s_cbranch_execnz .LBB24_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
+; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 255
- %unused = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret double %result
}
-define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB21_1
+; GFX12-NEXT: s_cbranch_execnz .LBB25_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX940-NEXT: s_nop 1
-; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: flat_load_b64 v[4:5], v[4:5]
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffff800, v0
-; GFX908-NEXT: s_mov_b64 s[4:5], vcc
-; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX908-NEXT: v_addc_co_u32_e64 v7, vcc, -1, v1, s[4:5]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[8:9]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[2:5] glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_1
+; GFX908-NEXT: s_cbranch_execnz .LBB25_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, -1, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[0:1]
-; GFX8-NEXT: flat_load_dword v4, v[6:7]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[0:1]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_1
+; GFX8-NEXT: s_cbranch_execnz .LBB25_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[2:3]
+; GFX7-NEXT: flat_atomic_fmax_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 -256
- %unused = atomicrmw fmax ptr %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret double %result
}
; --------------------------------------------------------------------
; half
; --------------------------------------------------------------------
-define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16:
+define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3658,7 +4248,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -3679,13 +4269,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB22_1
+; GFX12-NEXT: s_cbranch_execnz .LBB26_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -3698,7 +4288,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -3714,13 +4304,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB22_1
+; GFX940-NEXT: s_cbranch_execnz .LBB26_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -3734,7 +4324,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -3756,13 +4346,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -3774,7 +4364,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -3791,13 +4381,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB22_1
+; GFX10-NEXT: s_cbranch_execnz .LBB26_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -3810,7 +4400,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -3825,13 +4415,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -3844,7 +4434,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -3859,13 +4449,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB22_1
+; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -3878,7 +4468,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -3894,13 +4484,13 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB22_1
+; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -3913,7 +4503,7 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -3930,18 +4520,18 @@ define half @flat_agent_atomic_fmax_ret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB22_1
+; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3960,7 +4550,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -3981,13 +4571,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB23_1
+; GFX12-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -4002,7 +4592,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4018,13 +4608,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB23_1
+; GFX940-NEXT: s_cbranch_execnz .LBB27_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4039,7 +4629,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4061,13 +4651,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4080,7 +4670,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4097,13 +4687,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB23_1
+; GFX10-NEXT: s_cbranch_execnz .LBB27_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4117,7 +4707,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4132,13 +4722,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4152,7 +4742,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4167,13 +4757,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB23_1
+; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -4187,7 +4777,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4203,13 +4793,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB23_1
+; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -4223,7 +4813,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -4240,19 +4830,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB23_1
+; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %result = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4271,7 +4861,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -4292,13 +4882,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB24_1
+; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -4314,7 +4904,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4330,13 +4920,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB24_1
+; GFX940-NEXT: s_cbranch_execnz .LBB28_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4351,7 +4941,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4373,13 +4963,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4392,7 +4982,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4409,13 +4999,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB24_1
+; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4429,7 +5019,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4444,13 +5034,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4464,7 +5054,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4479,13 +5069,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_1
+; GFX908-NEXT: s_cbranch_execnz .LBB28_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4499,7 +5089,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4515,13 +5105,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB24_1
+; GFX8-NEXT: s_cbranch_execnz .LBB28_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v0
@@ -4535,7 +5125,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -4552,19 +5142,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB24_1
+; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 -1024
- %result = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16:
+define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4582,7 +5172,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4603,12 +5193,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB25_1
+; GFX12-NEXT: s_cbranch_execnz .LBB29_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -4621,7 +5211,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: v_not_b32_e32 v6, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -4637,12 +5227,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB25_1
+; GFX940-NEXT: s_cbranch_execnz .LBB29_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -4656,7 +5246,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4678,12 +5268,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-NEXT: s_cbranch_execnz .LBB29_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -4695,7 +5285,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4712,12 +5302,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB25_1
+; GFX10-NEXT: s_cbranch_execnz .LBB29_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -4730,7 +5320,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -4745,12 +5335,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -4763,7 +5353,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4778,12 +5368,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_1
+; GFX908-NEXT: s_cbranch_execnz .LBB29_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -4796,7 +5386,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4812,12 +5402,12 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB25_1
+; GFX8-NEXT: s_cbranch_execnz .LBB29_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -4830,7 +5420,7 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
@@ -4847,16 +5437,16 @@ define void @flat_agent_atomic_fmax_noret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB25_1
+; GFX7-NEXT: s_cbranch_execnz .LBB29_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4875,7 +5465,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -4896,12 +5486,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB26_1
+; GFX12-NEXT: s_cbranch_execnz .LBB30_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -4916,7 +5506,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -4932,12 +5522,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB26_1
+; GFX940-NEXT: s_cbranch_execnz .LBB30_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -4952,7 +5542,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -4974,12 +5564,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB26_1
+; GFX11-NEXT: s_cbranch_execnz .LBB30_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -4992,7 +5582,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5009,12 +5599,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB26_1
+; GFX10-NEXT: s_cbranch_execnz .LBB30_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5028,7 +5618,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5043,12 +5633,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5062,7 +5652,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5077,12 +5667,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB26_1
+; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -5096,7 +5686,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5112,12 +5702,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB26_1
+; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -5131,7 +5721,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5148,17 +5738,17 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB26_1
+; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5177,7 +5767,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5198,12 +5788,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB27_1
+; GFX12-NEXT: s_cbranch_execnz .LBB31_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -5219,7 +5809,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5235,12 +5825,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB27_1
+; GFX940-NEXT: s_cbranch_execnz .LBB31_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5255,7 +5845,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5277,12 +5867,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB27_1
+; GFX11-NEXT: s_cbranch_execnz .LBB31_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5295,7 +5885,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5312,12 +5902,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB27_1
+; GFX10-NEXT: s_cbranch_execnz .LBB31_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5331,7 +5921,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5346,12 +5936,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5365,7 +5955,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5380,12 +5970,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB27_1
+; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5399,7 +5989,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5415,12 +6005,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB27_1
+; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -5434,7 +6024,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5451,17 +6041,17 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB27_1
+; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 -1024
- %unused = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5471,7 +6061,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -5489,20 +6079,20 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB28_1
+; GFX12-NEXT: s_cbranch_execnz .LBB32_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -5516,19 +6106,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB28_1
+; GFX940-NEXT: s_cbranch_execnz .LBB32_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -5547,13 +6137,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB28_1
+; GFX11-NEXT: s_cbranch_execnz .LBB32_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -5561,7 +6151,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX10-NEXT: v_max_f16_e32 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -5577,19 +6167,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB28_1
+; GFX10-NEXT: s_cbranch_execnz .LBB32_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -5602,20 +6192,20 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -5628,13 +6218,13 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB28_1
+; GFX908-NEXT: s_cbranch_execnz .LBB32_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -5642,7 +6232,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -5656,12 +6246,12 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB28_1
+; GFX8-NEXT: s_cbranch_execnz .LBB32_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -5670,7 +6260,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v3
@@ -5685,18 +6275,18 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB28_1
+; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %result = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5706,7 +6296,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_max_num_f16_e32 v2, v3, v3
@@ -5724,19 +6314,19 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB29_1
+; GFX12-NEXT: s_cbranch_execnz .LBB33_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v4, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5750,18 +6340,18 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB29_1
+; GFX940-NEXT: s_cbranch_execnz .LBB33_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5780,12 +6370,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB29_1
+; GFX11-NEXT: s_cbranch_execnz .LBB33_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
@@ -5793,7 +6383,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5809,19 +6399,19 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB29_1
+; GFX10-NEXT: s_cbranch_execnz .LBB33_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5834,19 +6424,19 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5859,12 +6449,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB29_1
+; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -5872,7 +6462,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
-; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5886,12 +6476,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB29_1
+; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -5900,7 +6490,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -5915,17 +6505,17 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB29_1
+; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmax ptr %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5944,7 +6534,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -5965,13 +6555,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB30_1
+; GFX12-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -5986,7 +6576,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6002,13 +6592,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB30_1
+; GFX940-NEXT: s_cbranch_execnz .LBB34_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6023,7 +6613,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -6045,13 +6635,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB30_1
+; GFX11-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6064,7 +6654,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -6081,13 +6671,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB30_1
+; GFX10-NEXT: s_cbranch_execnz .LBB34_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6101,7 +6691,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -6118,13 +6708,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6138,7 +6728,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -6153,13 +6743,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_1
+; GFX908-NEXT: s_cbranch_execnz .LBB34_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -6173,7 +6763,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -6189,13 +6779,13 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB30_1
+; GFX8-NEXT: s_cbranch_execnz .LBB34_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -6209,7 +6799,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -6226,19 +6816,19 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB30_1
+; GFX7-NEXT: s_cbranch_execnz .LBB34_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %result = atomicrmw fmax ptr %gep, half %val seq_cst
+ %result = atomicrmw fmax ptr %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6257,7 +6847,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6278,12 +6868,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB31_1
+; GFX12-NEXT: s_cbranch_execnz .LBB35_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -6298,7 +6888,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6314,12 +6904,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB31_1
+; GFX940-NEXT: s_cbranch_execnz .LBB35_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -6334,7 +6924,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6356,12 +6946,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB31_1
+; GFX11-NEXT: s_cbranch_execnz .LBB35_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -6374,7 +6964,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6391,12 +6981,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB31_1
+; GFX10-NEXT: s_cbranch_execnz .LBB35_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -6410,7 +7000,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6427,12 +7017,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -6446,7 +7036,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6461,12 +7051,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB31_1
+; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -6480,7 +7070,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6496,12 +7086,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB31_1
+; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -6515,7 +7105,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6532,12 +7122,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB31_1
+; GFX7-NEXT: s_cbranch_execnz .LBB35_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, half %val seq_cst
+ %unused = atomicrmw fmax ptr %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -6545,8 +7135,8 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos(ptr %ptr, half %va
; bfloat
; --------------------------------------------------------------------
-define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16:
+define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6563,7 +7153,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -6591,13 +7181,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -6611,7 +7201,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6633,13 +7223,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB32_1
+; GFX940-NEXT: s_cbranch_execnz .LBB36_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -6653,7 +7243,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -6682,13 +7272,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -6700,7 +7290,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -6721,13 +7311,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB32_1
+; GFX10-NEXT: s_cbranch_execnz .LBB36_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -6741,7 +7331,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -6760,13 +7350,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -6780,7 +7370,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -6799,13 +7389,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB32_1
+; GFX908-NEXT: s_cbranch_execnz .LBB36_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -6818,7 +7408,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -6839,13 +7429,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB32_1
+; GFX8-NEXT: s_cbranch_execnz .LBB36_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -6858,7 +7448,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -6876,18 +7466,18 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB32_1
+; GFX7-NEXT: s_cbranch_execnz .LBB36_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6906,7 +7496,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -6934,13 +7524,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -6956,7 +7546,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6978,13 +7568,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB33_1
+; GFX940-NEXT: s_cbranch_execnz .LBB37_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7000,7 +7590,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7029,13 +7619,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7048,7 +7638,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7069,13 +7659,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB33_1
+; GFX10-NEXT: s_cbranch_execnz .LBB37_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7090,7 +7680,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7109,13 +7699,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7130,7 +7720,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7149,13 +7739,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB33_1
+; GFX908-NEXT: s_cbranch_execnz .LBB37_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -7169,7 +7759,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -7190,13 +7780,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB33_1
+; GFX8-NEXT: s_cbranch_execnz .LBB37_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -7210,7 +7800,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -7228,19 +7818,19 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB33_1
+; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7259,7 +7849,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -7287,13 +7877,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -7310,7 +7900,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -7332,13 +7922,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB34_1
+; GFX940-NEXT: s_cbranch_execnz .LBB38_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -7354,7 +7944,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7383,13 +7973,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -7402,7 +7992,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7423,13 +8013,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB34_1
+; GFX10-NEXT: s_cbranch_execnz .LBB38_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -7444,7 +8034,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7463,13 +8053,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -7484,7 +8074,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7503,13 +8093,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB34_1
+; GFX908-NEXT: s_cbranch_execnz .LBB38_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -7523,7 +8113,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -7544,13 +8134,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB34_1
+; GFX8-NEXT: s_cbranch_execnz .LBB38_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v0
@@ -7564,7 +8154,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -7582,19 +8172,19 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB34_1
+; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 -1024
- %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16:
+define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7611,7 +8201,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -7638,12 +8228,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -7657,7 +8247,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7679,12 +8269,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB35_1
+; GFX940-NEXT: s_cbranch_execnz .LBB39_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -7698,7 +8288,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -7726,12 +8316,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -7743,7 +8333,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7764,12 +8354,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB35_1
+; GFX10-NEXT: s_cbranch_execnz .LBB39_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -7783,7 +8373,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7802,12 +8392,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -7821,7 +8411,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7840,12 +8430,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB35_1
+; GFX908-NEXT: s_cbranch_execnz .LBB39_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -7858,7 +8448,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7879,12 +8469,12 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB35_1
+; GFX8-NEXT: s_cbranch_execnz .LBB39_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -7897,7 +8487,7 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -7915,16 +8505,16 @@ define void @flat_agent_atomic_fmax_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB35_1
+; GFX7-NEXT: s_cbranch_execnz .LBB39_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7943,7 +8533,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7970,12 +8560,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: s_cbranch_execnz .LBB40_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -7991,7 +8581,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8013,12 +8603,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB36_1
+; GFX940-NEXT: s_cbranch_execnz .LBB40_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -8034,7 +8624,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8062,12 +8652,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: s_cbranch_execnz .LBB40_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -8080,7 +8670,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8101,12 +8691,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB36_1
+; GFX10-NEXT: s_cbranch_execnz .LBB40_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -8121,7 +8711,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8140,12 +8730,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -8160,7 +8750,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8179,12 +8769,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB36_1
+; GFX908-NEXT: s_cbranch_execnz .LBB40_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -8198,7 +8788,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8219,12 +8809,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB36_1
+; GFX8-NEXT: s_cbranch_execnz .LBB40_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -8238,7 +8828,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8256,17 +8846,17 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB36_1
+; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8285,7 +8875,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8312,12 +8902,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: s_cbranch_execnz .LBB41_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -8334,7 +8924,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8356,12 +8946,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB37_1
+; GFX940-NEXT: s_cbranch_execnz .LBB41_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -8377,7 +8967,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8405,12 +8995,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: s_cbranch_execnz .LBB41_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -8423,7 +9013,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8444,12 +9034,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB37_1
+; GFX10-NEXT: s_cbranch_execnz .LBB41_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -8464,7 +9054,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8483,12 +9073,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -8503,7 +9093,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8522,12 +9112,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB37_1
+; GFX908-NEXT: s_cbranch_execnz .LBB41_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -8541,7 +9131,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8562,12 +9152,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB37_1
+; GFX8-NEXT: s_cbranch_execnz .LBB41_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -8581,7 +9171,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8599,17 +9189,17 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB37_1
+; GFX7-NEXT: s_cbranch_execnz .LBB41_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 -1024
- %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8619,7 +9209,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -8644,13 +9234,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8658,7 +9248,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -8679,20 +9269,20 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB38_1
+; GFX940-NEXT: s_cbranch_execnz .LBB42_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -8718,13 +9308,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -8732,7 +9322,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -8753,12 +9343,12 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB38_1
+; GFX10-NEXT: s_cbranch_execnz .LBB42_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8766,7 +9356,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -8785,13 +9375,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8799,7 +9389,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -8818,13 +9408,13 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB38_1
+; GFX908-NEXT: s_cbranch_execnz .LBB42_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -8832,7 +9422,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -8852,12 +9442,12 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB38_1
+; GFX8-NEXT: s_cbranch_execnz .LBB42_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -8866,7 +9456,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v3
@@ -8882,18 +9472,18 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB38_1
+; GFX7-NEXT: s_cbranch_execnz .LBB42_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8903,7 +9493,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -8927,12 +9517,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: s_cbranch_execnz .LBB43_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8940,7 +9530,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -8961,19 +9551,19 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB39_1
+; GFX940-NEXT: s_cbranch_execnz .LBB43_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -8998,12 +9588,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: s_cbranch_execnz .LBB43_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
@@ -9011,7 +9601,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9032,12 +9622,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB39_1
+; GFX10-NEXT: s_cbranch_execnz .LBB43_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -9045,7 +9635,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9064,12 +9654,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -9077,7 +9667,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9096,12 +9686,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB39_1
+; GFX908-NEXT: s_cbranch_execnz .LBB43_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -9109,7 +9699,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9129,12 +9719,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB39_1
+; GFX8-NEXT: s_cbranch_execnz .LBB43_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -9143,7 +9733,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9159,17 +9749,17 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB39_1
+; GFX7-NEXT: s_cbranch_execnz .LBB43_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9188,7 +9778,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -9216,13 +9806,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -9238,7 +9828,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -9260,13 +9850,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB40_1
+; GFX940-NEXT: s_cbranch_execnz .LBB44_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -9282,7 +9872,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -9311,13 +9901,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -9330,7 +9920,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -9351,13 +9941,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB40_1
+; GFX10-NEXT: s_cbranch_execnz .LBB44_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -9372,7 +9962,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -9393,13 +9983,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -9414,7 +10004,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -9433,13 +10023,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB40_1
+; GFX908-NEXT: s_cbranch_execnz .LBB44_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -9453,7 +10043,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -9474,13 +10064,13 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB40_1
+; GFX8-NEXT: s_cbranch_execnz .LBB44_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -9494,7 +10084,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -9512,19 +10102,19 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB40_1
+; GFX7-NEXT: s_cbranch_execnz .LBB44_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %result = atomicrmw fmax ptr %gep, bfloat %val seq_cst
+ %result = atomicrmw fmax ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9543,7 +10133,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9570,12 +10160,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB41_1
+; GFX12-NEXT: s_cbranch_execnz .LBB45_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -9591,7 +10181,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9613,12 +10203,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB41_1
+; GFX940-NEXT: s_cbranch_execnz .LBB45_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9634,7 +10224,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9662,12 +10252,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB41_1
+; GFX11-NEXT: s_cbranch_execnz .LBB45_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9680,7 +10270,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9701,12 +10291,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB41_1
+; GFX10-NEXT: s_cbranch_execnz .LBB45_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9721,7 +10311,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9742,12 +10332,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9762,7 +10352,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9781,12 +10371,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB41_1
+; GFX908-NEXT: s_cbranch_execnz .LBB45_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -9800,7 +10390,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9821,12 +10411,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB41_1
+; GFX8-NEXT: s_cbranch_execnz .LBB45_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -9840,7 +10430,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9858,12 +10448,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB41_1
+; GFX7-NEXT: s_cbranch_execnz .LBB45_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, bfloat %val seq_cst
+ %unused = atomicrmw fmax ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -9871,8 +10461,8 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; <2 x half>
; --------------------------------------------------------------------
-define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9882,7 +10472,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -9897,19 +10487,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB42_1
+; GFX12-NEXT: s_cbranch_execnz .LBB46_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -9923,19 +10513,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB42_1
+; GFX940-NEXT: s_cbranch_execnz .LBB46_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -9951,19 +10541,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB42_1
+; GFX11-NEXT: s_cbranch_execnz .LBB46_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -9977,19 +10567,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB42_1
+; GFX10-NEXT: s_cbranch_execnz .LBB46_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -10001,19 +10591,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -10025,20 +10615,20 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB42_1
+; GFX908-NEXT: s_cbranch_execnz .LBB46_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -10053,13 +10643,13 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB42_1
+; GFX8-NEXT: s_cbranch_execnz .LBB46_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2f16:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v5, v[0:1]
@@ -10072,7 +10662,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -10095,18 +10685,18 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB42_1
+; GFX7-NEXT: s_cbranch_execnz .LBB46_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10116,7 +10706,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -10131,19 +10721,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB43_1
+; GFX12-NEXT: s_cbranch_execnz .LBB47_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -10157,19 +10747,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB43_1
+; GFX940-NEXT: s_cbranch_execnz .LBB47_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -10185,13 +10775,13 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB43_1
+; GFX11-NEXT: s_cbranch_execnz .LBB47_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -10199,7 +10789,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -10213,18 +10803,18 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB43_1
+; GFX10-NEXT: s_cbranch_execnz .LBB47_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -10236,19 +10826,19 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -10260,13 +10850,13 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB43_1
+; GFX908-NEXT: s_cbranch_execnz .LBB47_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -10275,7 +10865,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -10290,12 +10880,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB43_1
+; GFX8-NEXT: s_cbranch_execnz .LBB47_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -10310,7 +10900,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -10333,17 +10923,17 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB43_1
+; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10353,7 +10943,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -10368,13 +10958,13 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: s_cbranch_execnz .LBB48_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v4, v0
@@ -10388,7 +10978,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -10402,12 +10992,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB44_1
+; GFX940-NEXT: s_cbranch_execnz .LBB48_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -10419,7 +11009,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX11-NEXT: flat_load_b32 v0, v[4:5]
; GFX11-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v0
@@ -10435,12 +11025,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: s_cbranch_execnz .LBB48_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -10448,7 +11038,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -10462,12 +11052,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB44_1
+; GFX10-NEXT: s_cbranch_execnz .LBB48_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -10477,7 +11067,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX90A-NEXT: flat_load_dword v0, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -10489,12 +11079,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -10504,7 +11094,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
@@ -10516,12 +11106,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB44_1
+; GFX908-NEXT: s_cbranch_execnz .LBB48_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -10530,7 +11120,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -10545,12 +11135,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB44_1
+; GFX8-NEXT: s_cbranch_execnz .LBB48_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -10565,7 +11155,7 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -10588,17 +11178,17 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB44_1
+; GFX7-NEXT: s_cbranch_execnz .LBB48_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 -512
- %result = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10608,7 +11198,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -10623,18 +11213,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB45_1
+; GFX12-NEXT: s_cbranch_execnz .LBB49_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10648,18 +11238,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB45_1
+; GFX940-NEXT: s_cbranch_execnz .LBB49_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10675,18 +11265,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB45_1
+; GFX11-NEXT: s_cbranch_execnz .LBB49_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10700,18 +11290,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB45_1
+; GFX10-NEXT: s_cbranch_execnz .LBB49_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10723,18 +11313,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10746,19 +11336,19 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB45_1
+; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -10773,12 +11363,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB45_1
+; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2f16:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v5, v[0:1]
@@ -10791,7 +11381,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -10814,16 +11404,16 @@ define void @flat_agent_atomic_fmax_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB45_1
+; GFX7-NEXT: s_cbranch_execnz .LBB49_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10833,7 +11423,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -10848,18 +11438,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-NEXT: s_cbranch_execnz .LBB50_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10873,18 +11463,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB46_1
+; GFX940-NEXT: s_cbranch_execnz .LBB50_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10900,12 +11490,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-NEXT: s_cbranch_execnz .LBB50_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -10913,7 +11503,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10927,18 +11517,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB46_1
+; GFX10-NEXT: s_cbranch_execnz .LBB50_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10950,18 +11540,18 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10973,12 +11563,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB46_1
+; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -10987,7 +11577,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -11002,12 +11592,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB46_1
+; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -11022,7 +11612,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11045,17 +11635,17 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB46_1
+; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11065,7 +11655,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -11080,12 +11670,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -11097,7 +11687,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX940-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11111,12 +11701,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB47_1
+; GFX940-NEXT: s_cbranch_execnz .LBB51_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -11126,7 +11716,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX11-NEXT: flat_load_b32 v3, v[3:4]
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11142,12 +11732,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -11155,7 +11745,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11169,12 +11759,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB47_1
+; GFX10-NEXT: s_cbranch_execnz .LBB51_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -11184,7 +11774,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
@@ -11196,12 +11786,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -11211,7 +11801,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
@@ -11223,12 +11813,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB47_1
+; GFX908-NEXT: s_cbranch_execnz .LBB51_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -11237,7 +11827,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -11252,12 +11842,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB47_1
+; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -11272,7 +11862,7 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11295,17 +11885,17 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB47_1
+; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 -512
- %unused = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11315,7 +11905,7 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11330,19 +11920,19 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-NEXT: s_cbranch_execnz .LBB52_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11356,19 +11946,19 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB48_1
+; GFX940-NEXT: s_cbranch_execnz .LBB52_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11384,13 +11974,13 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-NEXT: s_cbranch_execnz .LBB52_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -11398,7 +11988,7 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -11412,18 +12002,18 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB48_1
+; GFX10-NEXT: s_cbranch_execnz .LBB52_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11437,19 +12027,19 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11461,13 +12051,13 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB48_1
+; GFX908-NEXT: s_cbranch_execnz .LBB52_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -11476,7 +12066,7 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -11491,12 +12081,12 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB48_1
+; GFX8-NEXT: s_cbranch_execnz .LBB52_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -11511,7 +12101,7 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -11534,17 +12124,17 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB48_1
+; GFX7-NEXT: s_cbranch_execnz .LBB52_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, <2 x half> %val seq_cst
+ %result = atomicrmw fmax ptr %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11554,7 +12144,7 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -11569,18 +12159,18 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB49_1
+; GFX12-NEXT: s_cbranch_execnz .LBB53_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11594,18 +12184,18 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB49_1
+; GFX940-NEXT: s_cbranch_execnz .LBB53_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11621,12 +12211,12 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB49_1
+; GFX11-NEXT: s_cbranch_execnz .LBB53_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -11634,7 +12224,7 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11648,18 +12238,18 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB49_1
+; GFX10-NEXT: s_cbranch_execnz .LBB53_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11673,18 +12263,18 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11696,12 +12286,12 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB49_1
+; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -11710,7 +12300,7 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -11725,12 +12315,12 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB49_1
+; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -11745,7 +12335,7 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11768,12 +12358,12 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB49_1
+; GFX7-NEXT: s_cbranch_execnz .LBB53_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, <2 x half> %val seq_cst
+ %unused = atomicrmw fmax ptr %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -11781,8 +12371,8 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; <2 x bfloat>
; --------------------------------------------------------------------
-define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11793,7 +12383,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -11825,13 +12415,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB50_1
+; GFX12-NEXT: s_cbranch_execnz .LBB54_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -11840,7 +12430,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -11867,13 +12457,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB50_1
+; GFX940-NEXT: s_cbranch_execnz .LBB54_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
@@ -11882,7 +12472,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -11915,21 +12505,21 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB50_1
+; GFX11-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -11956,13 +12546,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB50_1
+; GFX10-NEXT: s_cbranch_execnz .LBB54_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -11971,7 +12561,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -11996,13 +12586,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -12011,7 +12601,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -12036,20 +12626,20 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB50_1
+; GFX908-NEXT: s_cbranch_execnz .LBB54_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -12077,13 +12667,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB50_1
+; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2bf16:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v5, v[0:1]
@@ -12095,7 +12685,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -12115,18 +12705,18 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB50_1
+; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12137,7 +12727,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -12169,13 +12759,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB51_1
+; GFX12-NEXT: s_cbranch_execnz .LBB55_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -12184,7 +12774,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -12211,13 +12801,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB51_1
+; GFX940-NEXT: s_cbranch_execnz .LBB55_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -12226,7 +12816,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -12259,14 +12849,14 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB51_1
+; GFX11-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -12275,7 +12865,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -12302,12 +12892,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB51_1
+; GFX10-NEXT: s_cbranch_execnz .LBB55_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -12316,7 +12906,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -12341,13 +12931,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -12356,7 +12946,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -12381,13 +12971,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB51_1
+; GFX908-NEXT: s_cbranch_execnz .LBB55_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -12396,7 +12986,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -12424,12 +13014,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB51_1
+; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -12443,7 +13033,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -12463,17 +13053,17 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB51_1
+; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12484,7 +13074,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -12516,13 +13106,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB52_1
+; GFX12-NEXT: s_cbranch_execnz .LBB56_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v4, v0
@@ -12539,7 +13129,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v0
@@ -12566,12 +13156,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB52_1
+; GFX940-NEXT: s_cbranch_execnz .LBB56_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -12586,7 +13176,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v0
@@ -12619,13 +13209,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB52_1
+; GFX11-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -12634,7 +13224,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -12661,12 +13251,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB52_1
+; GFX10-NEXT: s_cbranch_execnz .LBB56_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -12679,7 +13269,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
@@ -12704,12 +13294,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -12722,7 +13312,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
@@ -12747,12 +13337,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB52_1
+; GFX908-NEXT: s_cbranch_execnz .LBB56_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -12761,7 +13351,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -12789,12 +13379,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB52_1
+; GFX8-NEXT: s_cbranch_execnz .LBB56_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -12808,7 +13398,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -12828,17 +13418,17 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB52_1
+; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 -512
- %result = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12849,7 +13439,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -12880,12 +13470,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-NEXT: s_cbranch_execnz .LBB57_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -12894,7 +13484,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -12921,12 +13511,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB53_1
+; GFX940-NEXT: s_cbranch_execnz .LBB57_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
@@ -12935,7 +13525,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -12967,20 +13557,20 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13007,12 +13597,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB53_1
+; GFX10-NEXT: s_cbranch_execnz .LBB57_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -13021,7 +13611,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13046,12 +13636,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -13060,7 +13650,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13085,19 +13675,19 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB53_1
+; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13125,12 +13715,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB53_1
+; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v4, v[0:1]
@@ -13142,7 +13732,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -13162,16 +13752,16 @@ define void @flat_agent_atomic_fmax_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB53_1
+; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13182,7 +13772,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13213,12 +13803,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-NEXT: s_cbranch_execnz .LBB58_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13227,7 +13817,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13254,12 +13844,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB54_1
+; GFX940-NEXT: s_cbranch_execnz .LBB58_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -13268,7 +13858,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13300,13 +13890,13 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -13315,7 +13905,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13342,12 +13932,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB54_1
+; GFX10-NEXT: s_cbranch_execnz .LBB58_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13356,7 +13946,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13381,12 +13971,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13395,7 +13985,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13420,12 +14010,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB54_1
+; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -13434,7 +14024,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13462,12 +14052,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB54_1
+; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -13481,7 +14071,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -13501,17 +14091,17 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB54_1
+; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13522,7 +14112,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13553,12 +14143,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-NEXT: s_cbranch_execnz .LBB59_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -13573,7 +14163,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13600,12 +14190,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB55_1
+; GFX940-NEXT: s_cbranch_execnz .LBB59_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -13618,7 +14208,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13650,13 +14240,13 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -13665,7 +14255,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13692,12 +14282,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB55_1
+; GFX10-NEXT: s_cbranch_execnz .LBB59_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -13710,7 +14300,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -13735,12 +14325,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -13753,7 +14343,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -13778,12 +14368,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB55_1
+; GFX908-NEXT: s_cbranch_execnz .LBB59_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -13792,7 +14382,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13820,12 +14410,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB55_1
+; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -13839,7 +14429,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -13859,17 +14449,17 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB55_1
+; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 -512
- %unused = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13880,7 +14470,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -13912,13 +14502,13 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13927,7 +14517,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -13954,13 +14544,13 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB56_1
+; GFX940-NEXT: s_cbranch_execnz .LBB60_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -13969,7 +14559,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -14002,14 +14592,14 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -14018,7 +14608,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -14045,12 +14635,12 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB56_1
+; GFX10-NEXT: s_cbranch_execnz .LBB60_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14059,7 +14649,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -14086,13 +14676,13 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14101,7 +14691,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -14126,13 +14716,13 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB56_1
+; GFX908-NEXT: s_cbranch_execnz .LBB60_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -14141,7 +14731,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -14169,12 +14759,12 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB56_1
+; GFX8-NEXT: s_cbranch_execnz .LBB60_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -14188,7 +14778,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14208,17 +14798,17 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB56_1
+; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %result = atomicrmw fmax ptr %gep, <2 x bfloat> %val seq_cst
+ %result = atomicrmw fmax ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -14229,7 +14819,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14260,12 +14850,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-NEXT: s_cbranch_execnz .LBB61_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14274,7 +14864,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14301,12 +14891,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB57_1
+; GFX940-NEXT: s_cbranch_execnz .LBB61_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -14315,7 +14905,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14347,13 +14937,13 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -14362,7 +14952,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14389,12 +14979,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB57_1
+; GFX10-NEXT: s_cbranch_execnz .LBB61_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14403,7 +14993,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14430,12 +15020,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14444,7 +15034,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14469,12 +15059,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB57_1
+; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -14483,7 +15073,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14511,12 +15101,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB57_1
+; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -14530,7 +15120,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -14550,14 +15140,16 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB57_1
+; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %unused = atomicrmw fmax ptr %gep, <2 x bfloat> %val seq_cst
+ %unused = atomicrmw fmax ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 216ea07833ff8..6babcfd15ee1a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -12,8 +12,8 @@
; float
; --------------------------------------------------------------------
-define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32:
+define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -26,7 +26,7 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -51,7 +51,7 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -61,7 +61,7 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -71,7 +71,7 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -95,7 +95,7 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -119,7 +119,7 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -143,19 +143,19 @@ define float @flat_agent_atomic_fmin_ret_f32(ptr %ptr, float %val) #0 {
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -168,7 +168,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -193,7 +193,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -203,7 +203,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -215,7 +215,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -239,7 +239,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -263,7 +263,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -288,7 +288,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -298,12 +298,12 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -316,7 +316,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v4, v0
@@ -348,7 +348,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -360,7 +360,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -372,7 +372,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -399,7 +399,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -426,7 +426,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -451,7 +451,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -461,12 +461,12 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 -512
- %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32:
+define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -479,7 +479,7 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -503,7 +503,7 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -514,7 +514,7 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -525,7 +525,7 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -548,7 +548,7 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -571,7 +571,7 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -594,19 +594,19 @@ define void @flat_agent_atomic_fmin_noret_f32(ptr %ptr, float %val) #0 {
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_atomic_fmin v[0:1], v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -619,7 +619,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -643,7 +643,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -654,7 +654,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -667,7 +667,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -690,7 +690,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -713,7 +713,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -738,7 +738,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -748,12 +748,12 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %va
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -766,7 +766,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -796,7 +796,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -809,7 +809,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -822,7 +822,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -849,7 +849,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -876,7 +876,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -901,7 +901,7 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -911,12 +911,12 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 -512
- %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -947,7 +947,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -972,7 +972,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -1000,7 +1000,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -1027,7 +1027,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1053,7 +1053,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1077,7 +1077,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -1102,7 +1102,7 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fc, v0
@@ -1127,12 +1127,12 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos(ptr %ptr, float %va
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, float %val seq_cst
+ %result = atomicrmw fmin ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1162,7 +1162,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1186,7 +1186,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -1213,7 +1213,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -1240,7 +1240,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1265,7 +1265,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -1288,7 +1288,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -1313,7 +1313,7 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -1338,16 +1338,12 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos(ptr %ptr, float %v
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, float %val seq_cst
+ %unused = atomicrmw fmin ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-; --------------------------------------------------------------------
-; float with ftz/daz
-; --------------------------------------------------------------------
-
-define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1360,7 +1356,7 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -1385,7 +1381,7 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1395,7 +1391,7 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1405,7 +1401,7 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -1429,7 +1425,7 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -1453,7 +1449,7 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -1477,19 +1473,19 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr %ptr, float %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1497,15 +1493,15 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_min_num_f32 v0, v[0:1], v2 offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_min_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1515,7 +1511,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1527,21 +1523,19 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_min_f32 v0, v[0:1], v2 offset:2044 glc
+; GFX11-NEXT: flat_atomic_min_f32 v0, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1549,10 +1543,10 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1561,7 +1555,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1573,10 +1567,10 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1585,7 +1579,7 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -1597,47 +1591,47 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; --------------------------------------------------------------------
+; float with ftz/daz
+; --------------------------------------------------------------------
+
+define float @flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1645,48 +1639,39 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_min_num_f32 v0, v[0:1], v2 offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_min_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v4, v0
-; GFX940-NEXT: v_mov_b32_e32 v5, v1
-; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v4
-; GFX940-NEXT: s_movk_i32 s0, 0xf800
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v5, vcc
-; GFX940-NEXT: flat_load_dword v0, v[0:1]
-; GFX940-NEXT: s_mov_b32 s1, -1
-; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[0:1]
+; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v1, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v3, v0
-; GFX940-NEXT: v_max_f32_e32 v0, v3, v3
-; GFX940-NEXT: v_min_f32_e32 v2, v0, v1
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB10_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_min_f32 v0, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1694,11 +1679,9 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1706,101 +1689,91 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_load_dword v0, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v0, v1
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: flat_load_dword v0, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v6, v0
-; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
-; GFX908-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 -512
- %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @flat_agent_atomic_fmin_noret_f32__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1808,139 +1781,147 @@ define void @flat_agent_atomic_fmin_noret_f32__ftz(ptr %ptr, float %val) #1 {
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_min_num_f32 v[0:1], v2
-; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: flat_atomic_min_num_f32 v0, v[0:1], v2 offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_load_dword v3, v[0:1]
+; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB11_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_min_f32 v[0:1], v2
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_min_f32 v0, v[0:1], v2 offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmin v[0:1], v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_atomic_fmin v[0:1], v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr %ptr, i64 511
+ %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1948,146 +1929,162 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr %ptr, floa
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_min_num_f32 v[0:1], v2 offset:2044
-; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: flat_atomic_min_num_f32 v0, v[0:1], v2 offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX940-NEXT: v_mov_b32_e32 v4, v0
+; GFX940-NEXT: v_mov_b32_e32 v5, v1
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v4
+; GFX940-NEXT: s_movk_i32 s0, 0xf800
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v5, vcc
+; GFX940-NEXT: flat_load_dword v0, v[0:1]
+; GFX940-NEXT: s_mov_b32 s1, -1
+; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v1, v2, v2
; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v3, v0
+; GFX940-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX940-NEXT: v_min_f32_e32 v2, v0, v1
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
+; GFX940-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB12_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_min_f32 v[0:1], v2 offset:2044
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_min_f32 v0, v[0:1], v2 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmin v[0:1], v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_load_dword v0, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v0, v1
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
+; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX908-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_atomic_fmin v[0:1], v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX7-NEXT: flat_atomic_fmin v0, v[0:1], v2 glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr %ptr, i64 -512
+ %result = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+define void @flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2095,21 +2092,15 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_min_num_f32 v[0:1], v2 offset:-2048
+; GFX12-NEXT: flat_atomic_min_num_f32 v[0:1], v2
; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX940-NEXT: s_movk_i32 s0, 0xf800
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX940-NEXT: flat_load_dword v3, v[4:5]
-; GFX940-NEXT: s_mov_b32 s1, -1
-; GFX940-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2130,11 +2121,9 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_min_f32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -2143,11 +2132,9 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmin v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -2156,65 +2143,55 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_load_dword v1, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: flat_load_dword v1, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
@@ -2235,238 +2212,329 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmin v[0:1], v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 -512
- %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-NEXT: flat_atomic_min_num_f32 v[0:1], v2 offset:2044
+; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v5, v3
-; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: buffer_inv sc0 sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB14_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_atomic_min_f32 v[0:1], v2 offset:2044
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v6, v0
-; GFX10-NEXT: v_max_f32_e32 v0, v6, v6
-; GFX10-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin v[0:1], v2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_1
-; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fc, v0
-; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v0, v[3:4]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX7-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX7-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_atomic_fmin v[0:1], v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, float %val seq_cst
- ret float %result
+ %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_min_num_f32 v[0:1], v2 offset:-2048
+; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_movk_i32 s0, 0xf800
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_load_dword v3, v[4:5]
+; GFX940-NEXT: s_mov_b32 s1, -1
+; GFX940-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB15_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_min_f32 v[0:1], v2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin v[0:1], v2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_load_dword v1, v[0:1]
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
+; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT: flat_load_dword v1, v[0:1]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB15_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB15_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX7-NEXT: flat_atomic_fmin v[0:1], v2
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr %ptr, i64 -512
+ %unused = atomicrmw fmin ptr %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr %ptr, float %val) #1 {
-; GFX12-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2474,224 +2542,751 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr %ptr, flo
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
+; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc0 sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB15_1
+; GFX940-NEXT: s_cbranch_execnz .LBB16_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
-; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f32_e32 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX10-NEXT: flat_load_dword v0, v[3:4]
+; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX10-NEXT: v_min_f32_e32 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB15_1
+; GFX10-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_load_dword v0, v[3:4]
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX7-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX7-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB16_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr %ptr, i64 511
+ %result = atomicrmw fmin ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
+}
+
+define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr %ptr, float %val) #1 {
+; GFX12-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc0 sc1
+; GFX940-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB17_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB17_1
+; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_load_dword v3, v[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX7-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v3, v2
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr %ptr, i64 511
+ %unused = atomicrmw fmin ptr %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+; --------------------------------------------------------------------
+; double
+; --------------------------------------------------------------------
+
+define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] offset:2040 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] offset:2040 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
-; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB15_1
+; GFX908-NEXT: s_cbranch_execnz .LBB19_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[0:1]
+; GFX8-NEXT: flat_load_dword v0, v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB15_1
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX7-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v3, v2
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB15_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, float %val seq_cst
- ret void
+ %gep = getelementptr double, ptr %ptr, i64 255
+ %result = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
}
-; --------------------------------------------------------------------
-; double
-; --------------------------------------------------------------------
-
-define double @flat_agent_atomic_fmin_ret_f64(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64:
+define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
@@ -2699,59 +3294,69 @@ define double @flat_agent_atomic_fmin_ret_f64(ptr %ptr, double %val) #0 {
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: s_cbranch_execnz .LBB20_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v0
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v5
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v4, vcc_lo
+; GFX11-NEXT: v_add_co_u32 v5, vcc_lo, 0xfffff800, v5
+; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, -1, v4, vcc_lo
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[5:6], v[7:10] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: s_cbranch_execnz .LBB20_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2759,429 +3364,411 @@ define double @flat_agent_atomic_fmin_ret_f64(ptr %ptr, double %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_add_co_u32_e32 v5, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v1
+; GFX908-NEXT: s_mov_b64 s[4:5], vcc
+; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v4, vcc
+; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: v_addc_co_u32_e64 v6, vcc, -1, v4, s[4:5]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_mov_b32_e32 v10, v1
+; GFX908-NEXT: v_mov_b32_e32 v9, v0
+; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
+; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[5:6], v[7:10] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v1, v[0:1]
+; GFX8-NEXT: flat_load_dword v0, v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr %ptr, i64 -256
+ %result = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret double %result
}
-define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] offset:2040 sc0
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:2040 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] offset:2040 glc
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:2040 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v1, v[0:1]
-; GFX8-NEXT: flat_load_dword v0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 255
- %result = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %unused = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: s_cbranch_execnz .LBB22_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX940-NEXT: s_nop 1
-; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3] offset:2040
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v0
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v5
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v4, vcc_lo
-; GFX11-NEXT: v_add_co_u32 v5, vcc_lo, 0xfffff800, v5
-; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, -1, v4, vcc_lo
-; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1]
-; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[5:6], v[7:10] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: s_cbranch_execnz .LBB22_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3] offset:2040
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v5, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_mov_b32_e32 v4, v1
-; GFX908-NEXT: s_mov_b64 s[4:5], vcc
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v4, vcc
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_addc_co_u32_e64 v6, vcc, -1, v4, s[4:5]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v10, v1
-; GFX908-NEXT: v_mov_b32_e32 v9, v0
-; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
-; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[5:6], v[7:10] glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v1, v[0:1]
-; GFX8-NEXT: flat_load_dword v0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v5, v[0:1]
+; GFX8-NEXT: flat_load_dword v4, v[6:7]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 -256
- %result = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %gep = getelementptr double, ptr %ptr, i64 255
+ %unused = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @flat_agent_atomic_fmin_noret_f64(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f64:
+define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
@@ -3189,27 +3776,34 @@ define void @flat_agent_atomic_fmin_noret_f64(ptr %ptr, double %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: s_cbranch_execnz .LBB23_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: flat_load_b64 v[4:5], v[4:5]
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
@@ -3225,14 +3819,16 @@ define void @flat_agent_atomic_fmin_noret_f64(ptr %ptr, double %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: s_cbranch_execnz .LBB23_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -3241,53 +3837,62 @@ define void @flat_agent_atomic_fmin_noret_f64(ptr %ptr, double %val) #0 {
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffff800, v0
+; GFX908-NEXT: s_mov_b64 s[4:5], vcc
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
+; GFX908-NEXT: v_addc_co_u32_e64 v7, vcc, -1, v1, s[4:5]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[8:9]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB19_1
+; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: flat_load_dword v5, v[5:6]
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, -1, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v5, v[0:1]
+; GFX8-NEXT: flat_load_dword v4, v[6:7]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
@@ -3295,352 +3900,337 @@ define void @flat_agent_atomic_fmin_noret_f64(ptr %ptr, double %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr %ptr, i64 -256
+ %unused = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB20_1
+; GFX12-NEXT: s_cbranch_execnz .LBB24_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3] offset:2040
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1] offset:2040
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:2040 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB20_1
+; GFX11-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3] offset:2040
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:2040
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] offset:2040 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB20_1
+; GFX908-NEXT: s_cbranch_execnz .LBB24_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[0:1]
-; GFX8-NEXT: flat_load_dword v4, v[6:7]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[0:1]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB20_1
+; GFX8-NEXT: s_cbranch_execnz .LBB24_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7f8, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
+; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 255
- %unused = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret double %result
}
-define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg(ptr %ptr, double %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr %ptr, double %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1] offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB21_1
+; GFX12-NEXT: s_cbranch_execnz .LBB25_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX940-NEXT: s_nop 1
-; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: flat_load_b64 v[4:5], v[4:5]
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffff800, v0
-; GFX908-NEXT: s_mov_b64 s[4:5], vcc
-; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX908-NEXT: v_addc_co_u32_e64 v7, vcc, -1, v1, s[4:5]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[8:9]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[2:5] glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_1
+; GFX908-NEXT: s_cbranch_execnz .LBB25_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, -1, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff804, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[0:1]
-; GFX8-NEXT: flat_load_dword v4, v[6:7]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[0:1]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_1
+; GFX8-NEXT: s_cbranch_execnz .LBB25_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[2:3]
+; GFX7-NEXT: flat_atomic_fmin_x2 v[0:1], v[0:1], v[2:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr %ptr, i64 -256
- %unused = atomicrmw fmin ptr %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret double %result
}
; --------------------------------------------------------------------
; half
; --------------------------------------------------------------------
-define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16:
+define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3658,7 +4248,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -3679,13 +4269,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB22_1
+; GFX12-NEXT: s_cbranch_execnz .LBB26_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -3698,7 +4288,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -3714,13 +4304,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB22_1
+; GFX940-NEXT: s_cbranch_execnz .LBB26_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -3734,7 +4324,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -3756,13 +4346,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -3774,7 +4364,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -3791,13 +4381,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB22_1
+; GFX10-NEXT: s_cbranch_execnz .LBB26_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -3810,7 +4400,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -3825,13 +4415,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -3844,7 +4434,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -3859,13 +4449,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB22_1
+; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -3878,7 +4468,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -3894,13 +4484,13 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB22_1
+; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -3913,7 +4503,7 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -3930,18 +4520,18 @@ define half @flat_agent_atomic_fmin_ret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB22_1
+; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3960,7 +4550,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -3981,13 +4571,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB23_1
+; GFX12-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -4002,7 +4592,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4018,13 +4608,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB23_1
+; GFX940-NEXT: s_cbranch_execnz .LBB27_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4039,7 +4629,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4061,13 +4651,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4080,7 +4670,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4097,13 +4687,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB23_1
+; GFX10-NEXT: s_cbranch_execnz .LBB27_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4117,7 +4707,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4132,13 +4722,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4152,7 +4742,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4167,13 +4757,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB23_1
+; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -4187,7 +4777,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4203,13 +4793,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB23_1
+; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -4223,7 +4813,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -4240,19 +4830,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB23_1
+; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %result = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4271,7 +4861,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -4292,13 +4882,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB24_1
+; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -4314,7 +4904,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4330,13 +4920,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB24_1
+; GFX940-NEXT: s_cbranch_execnz .LBB28_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4351,7 +4941,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4373,13 +4963,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4392,7 +4982,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4409,13 +4999,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB24_1
+; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4429,7 +5019,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4444,13 +5034,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4464,7 +5054,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4479,13 +5069,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_1
+; GFX908-NEXT: s_cbranch_execnz .LBB28_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4499,7 +5089,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4515,13 +5105,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB24_1
+; GFX8-NEXT: s_cbranch_execnz .LBB28_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v0
@@ -4535,7 +5125,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -4552,19 +5142,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB24_1
+; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 -1024
- %result = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16:
+define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4582,7 +5172,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4603,12 +5193,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB25_1
+; GFX12-NEXT: s_cbranch_execnz .LBB29_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -4621,7 +5211,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: v_not_b32_e32 v6, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -4637,12 +5227,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB25_1
+; GFX940-NEXT: s_cbranch_execnz .LBB29_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -4656,7 +5246,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4678,12 +5268,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-NEXT: s_cbranch_execnz .LBB29_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -4695,7 +5285,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4712,12 +5302,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB25_1
+; GFX10-NEXT: s_cbranch_execnz .LBB29_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -4730,7 +5320,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -4745,12 +5335,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -4763,7 +5353,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4778,12 +5368,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_1
+; GFX908-NEXT: s_cbranch_execnz .LBB29_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -4796,7 +5386,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -4812,12 +5402,12 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB25_1
+; GFX8-NEXT: s_cbranch_execnz .LBB29_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -4830,7 +5420,7 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
@@ -4847,16 +5437,16 @@ define void @flat_agent_atomic_fmin_noret_f16(ptr %ptr, half %val) #0 {
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB25_1
+; GFX7-NEXT: s_cbranch_execnz .LBB29_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4875,7 +5465,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -4896,12 +5486,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB26_1
+; GFX12-NEXT: s_cbranch_execnz .LBB30_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -4916,7 +5506,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -4932,12 +5522,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB26_1
+; GFX940-NEXT: s_cbranch_execnz .LBB30_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -4952,7 +5542,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -4974,12 +5564,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB26_1
+; GFX11-NEXT: s_cbranch_execnz .LBB30_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -4992,7 +5582,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5009,12 +5599,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB26_1
+; GFX10-NEXT: s_cbranch_execnz .LBB30_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5028,7 +5618,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5043,12 +5633,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5062,7 +5652,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5077,12 +5667,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB26_1
+; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -5096,7 +5686,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5112,12 +5702,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB26_1
+; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -5131,7 +5721,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5148,17 +5738,17 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB26_1
+; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5177,7 +5767,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5198,12 +5788,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB27_1
+; GFX12-NEXT: s_cbranch_execnz .LBB31_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -5219,7 +5809,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5235,12 +5825,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB27_1
+; GFX940-NEXT: s_cbranch_execnz .LBB31_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5255,7 +5845,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5277,12 +5867,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB27_1
+; GFX11-NEXT: s_cbranch_execnz .LBB31_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5295,7 +5885,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5312,12 +5902,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB27_1
+; GFX10-NEXT: s_cbranch_execnz .LBB31_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5331,7 +5921,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5346,12 +5936,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5365,7 +5955,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5380,12 +5970,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB27_1
+; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5399,7 +5989,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5415,12 +6005,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB27_1
+; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -5434,7 +6024,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5451,17 +6041,17 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB27_1
+; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 -1024
- %unused = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5471,7 +6061,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -5489,20 +6079,20 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB28_1
+; GFX12-NEXT: s_cbranch_execnz .LBB32_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -5516,19 +6106,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB28_1
+; GFX940-NEXT: s_cbranch_execnz .LBB32_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -5547,13 +6137,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB28_1
+; GFX11-NEXT: s_cbranch_execnz .LBB32_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -5561,7 +6151,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX10-NEXT: v_max_f16_e32 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -5577,19 +6167,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB28_1
+; GFX10-NEXT: s_cbranch_execnz .LBB32_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -5602,20 +6192,20 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -5628,13 +6218,13 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB28_1
+; GFX908-NEXT: s_cbranch_execnz .LBB32_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -5642,7 +6232,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -5656,12 +6246,12 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB28_1
+; GFX8-NEXT: s_cbranch_execnz .LBB32_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -5670,7 +6260,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v3
@@ -5685,18 +6275,18 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB28_1
+; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %result = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5706,7 +6296,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_max_num_f16_e32 v2, v3, v3
@@ -5724,19 +6314,19 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB29_1
+; GFX12-NEXT: s_cbranch_execnz .LBB33_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v4, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5750,18 +6340,18 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB29_1
+; GFX940-NEXT: s_cbranch_execnz .LBB33_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5780,12 +6370,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB29_1
+; GFX11-NEXT: s_cbranch_execnz .LBB33_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
@@ -5793,7 +6383,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5809,19 +6399,19 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB29_1
+; GFX10-NEXT: s_cbranch_execnz .LBB33_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5834,19 +6424,19 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5859,12 +6449,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB29_1
+; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -5872,7 +6462,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
-; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
@@ -5886,12 +6476,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB29_1
+; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -5900,7 +6490,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -5915,17 +6505,17 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB29_1
+; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmin ptr %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5944,7 +6534,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -5965,13 +6555,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB30_1
+; GFX12-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -5986,7 +6576,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6002,13 +6592,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB30_1
+; GFX940-NEXT: s_cbranch_execnz .LBB34_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6023,7 +6613,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -6045,13 +6635,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB30_1
+; GFX11-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6064,7 +6654,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -6081,13 +6671,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB30_1
+; GFX10-NEXT: s_cbranch_execnz .LBB34_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6101,7 +6691,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -6118,13 +6708,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6138,7 +6728,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -6153,13 +6743,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_1
+; GFX908-NEXT: s_cbranch_execnz .LBB34_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -6173,7 +6763,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -6189,13 +6779,13 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB30_1
+; GFX8-NEXT: s_cbranch_execnz .LBB34_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -6209,7 +6799,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v4, v4
-; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -6226,19 +6816,19 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB30_1
+; GFX7-NEXT: s_cbranch_execnz .LBB34_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %result = atomicrmw fmin ptr %gep, half %val seq_cst
+ %result = atomicrmw fmin ptr %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, half %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6257,7 +6847,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6278,12 +6868,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB31_1
+; GFX12-NEXT: s_cbranch_execnz .LBB35_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -6298,7 +6888,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6314,12 +6904,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB31_1
+; GFX940-NEXT: s_cbranch_execnz .LBB35_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -6334,7 +6924,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6356,12 +6946,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB31_1
+; GFX11-NEXT: s_cbranch_execnz .LBB35_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -6374,7 +6964,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6391,12 +6981,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB31_1
+; GFX10-NEXT: s_cbranch_execnz .LBB35_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -6410,7 +7000,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6427,12 +7017,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -6446,7 +7036,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6461,12 +7051,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB31_1
+; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -6480,7 +7070,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6496,12 +7086,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB31_1
+; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -6515,7 +7105,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6532,12 +7122,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB31_1
+; GFX7-NEXT: s_cbranch_execnz .LBB35_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, half %val seq_cst
+ %unused = atomicrmw fmin ptr %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -6545,8 +7135,8 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos(ptr %ptr, half %va
; bfloat
; --------------------------------------------------------------------
-define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16:
+define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6563,7 +7153,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -6591,13 +7181,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -6611,7 +7201,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6633,13 +7223,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB32_1
+; GFX940-NEXT: s_cbranch_execnz .LBB36_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -6653,7 +7243,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -6682,13 +7272,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -6700,7 +7290,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -6721,13 +7311,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB32_1
+; GFX10-NEXT: s_cbranch_execnz .LBB36_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -6741,7 +7331,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -6760,13 +7350,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -6780,7 +7370,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -6799,13 +7389,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB32_1
+; GFX908-NEXT: s_cbranch_execnz .LBB36_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -6818,7 +7408,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -6839,13 +7429,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB32_1
+; GFX8-NEXT: s_cbranch_execnz .LBB36_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -6858,7 +7448,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -6876,18 +7466,18 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB32_1
+; GFX7-NEXT: s_cbranch_execnz .LBB36_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6906,7 +7496,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -6934,13 +7524,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -6956,7 +7546,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6978,13 +7568,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB33_1
+; GFX940-NEXT: s_cbranch_execnz .LBB37_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7000,7 +7590,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7029,13 +7619,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7048,7 +7638,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7069,13 +7659,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB33_1
+; GFX10-NEXT: s_cbranch_execnz .LBB37_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7090,7 +7680,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7109,13 +7699,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7130,7 +7720,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7149,13 +7739,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB33_1
+; GFX908-NEXT: s_cbranch_execnz .LBB37_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -7169,7 +7759,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -7190,13 +7780,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB33_1
+; GFX8-NEXT: s_cbranch_execnz .LBB37_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -7210,7 +7800,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -7228,19 +7818,19 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB33_1
+; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7259,7 +7849,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -7287,13 +7877,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -7310,7 +7900,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -7332,13 +7922,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB34_1
+; GFX940-NEXT: s_cbranch_execnz .LBB38_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -7354,7 +7944,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7383,13 +7973,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -7402,7 +7992,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7423,13 +8013,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB34_1
+; GFX10-NEXT: s_cbranch_execnz .LBB38_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -7444,7 +8034,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7463,13 +8053,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -7484,7 +8074,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7503,13 +8093,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB34_1
+; GFX908-NEXT: s_cbranch_execnz .LBB38_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -7523,7 +8113,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -7544,13 +8134,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB34_1
+; GFX8-NEXT: s_cbranch_execnz .LBB38_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v0
@@ -7564,7 +8154,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -7582,19 +8172,19 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB34_1
+; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 -1024
- %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16:
+define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7611,7 +8201,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -7638,12 +8228,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -7657,7 +8247,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7679,12 +8269,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB35_1
+; GFX940-NEXT: s_cbranch_execnz .LBB39_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -7698,7 +8288,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -7726,12 +8316,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -7743,7 +8333,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7764,12 +8354,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB35_1
+; GFX10-NEXT: s_cbranch_execnz .LBB39_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -7783,7 +8373,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7802,12 +8392,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -7821,7 +8411,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7840,12 +8430,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB35_1
+; GFX908-NEXT: s_cbranch_execnz .LBB39_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -7858,7 +8448,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7879,12 +8469,12 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB35_1
+; GFX8-NEXT: s_cbranch_execnz .LBB39_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -7897,7 +8487,7 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -7915,16 +8505,16 @@ define void @flat_agent_atomic_fmin_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB35_1
+; GFX7-NEXT: s_cbranch_execnz .LBB39_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7943,7 +8533,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7970,12 +8560,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: s_cbranch_execnz .LBB40_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -7991,7 +8581,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8013,12 +8603,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB36_1
+; GFX940-NEXT: s_cbranch_execnz .LBB40_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -8034,7 +8624,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8062,12 +8652,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: s_cbranch_execnz .LBB40_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -8080,7 +8670,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8101,12 +8691,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB36_1
+; GFX10-NEXT: s_cbranch_execnz .LBB40_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -8121,7 +8711,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8140,12 +8730,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -8160,7 +8750,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8179,12 +8769,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB36_1
+; GFX908-NEXT: s_cbranch_execnz .LBB40_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -8198,7 +8788,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8219,12 +8809,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB36_1
+; GFX8-NEXT: s_cbranch_execnz .LBB40_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -8238,7 +8828,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8256,17 +8846,17 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB36_1
+; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8285,7 +8875,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8312,12 +8902,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: s_cbranch_execnz .LBB41_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -8334,7 +8924,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8356,12 +8946,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB37_1
+; GFX940-NEXT: s_cbranch_execnz .LBB41_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -8377,7 +8967,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8405,12 +8995,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: s_cbranch_execnz .LBB41_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -8423,7 +9013,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8444,12 +9034,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB37_1
+; GFX10-NEXT: s_cbranch_execnz .LBB41_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -8464,7 +9054,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8483,12 +9073,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -8503,7 +9093,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8522,12 +9112,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB37_1
+; GFX908-NEXT: s_cbranch_execnz .LBB41_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -8541,7 +9131,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8562,12 +9152,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB37_1
+; GFX8-NEXT: s_cbranch_execnz .LBB41_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -8581,7 +9171,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8599,17 +9189,17 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB37_1
+; GFX7-NEXT: s_cbranch_execnz .LBB41_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 -1024
- %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8619,7 +9209,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -8644,13 +9234,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8658,7 +9248,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -8679,20 +9269,20 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB38_1
+; GFX940-NEXT: s_cbranch_execnz .LBB42_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -8718,13 +9308,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -8732,7 +9322,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -8753,12 +9343,12 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB38_1
+; GFX10-NEXT: s_cbranch_execnz .LBB42_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8766,7 +9356,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -8785,13 +9375,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8799,7 +9389,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -8818,13 +9408,13 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB38_1
+; GFX908-NEXT: s_cbranch_execnz .LBB42_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -8832,7 +9422,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -8852,12 +9442,12 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB38_1
+; GFX8-NEXT: s_cbranch_execnz .LBB42_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -8866,7 +9456,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v3
@@ -8882,18 +9472,18 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB38_1
+; GFX7-NEXT: s_cbranch_execnz .LBB42_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8903,7 +9493,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -8927,12 +9517,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: s_cbranch_execnz .LBB43_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -8940,7 +9530,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -8961,19 +9551,19 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB39_1
+; GFX940-NEXT: s_cbranch_execnz .LBB43_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -8998,12 +9588,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: s_cbranch_execnz .LBB43_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
@@ -9011,7 +9601,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9032,12 +9622,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB39_1
+; GFX10-NEXT: s_cbranch_execnz .LBB43_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -9045,7 +9635,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9064,12 +9654,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
@@ -9077,7 +9667,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9096,12 +9686,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB39_1
+; GFX908-NEXT: s_cbranch_execnz .LBB43_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -9109,7 +9699,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9129,12 +9719,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB39_1
+; GFX8-NEXT: s_cbranch_execnz .LBB43_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
@@ -9143,7 +9733,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9159,17 +9749,17 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB39_1
+; GFX7-NEXT: s_cbranch_execnz .LBB43_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9188,7 +9778,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -9216,13 +9806,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -9238,7 +9828,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -9260,13 +9850,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB40_1
+; GFX940-NEXT: s_cbranch_execnz .LBB44_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -9282,7 +9872,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -9311,13 +9901,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -9330,7 +9920,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -9351,13 +9941,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB40_1
+; GFX10-NEXT: s_cbranch_execnz .LBB44_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -9372,7 +9962,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -9393,13 +9983,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -9414,7 +10004,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -9433,13 +10023,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB40_1
+; GFX908-NEXT: s_cbranch_execnz .LBB44_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -9453,7 +10043,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -9474,13 +10064,13 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB40_1
+; GFX8-NEXT: s_cbranch_execnz .LBB44_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
@@ -9494,7 +10084,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
@@ -9512,19 +10102,19 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB40_1
+; GFX7-NEXT: s_cbranch_execnz .LBB44_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %result = atomicrmw fmin ptr %gep, bfloat %val seq_cst
+ %result = atomicrmw fmin ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9543,7 +10133,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9570,12 +10160,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB41_1
+; GFX12-NEXT: s_cbranch_execnz .LBB45_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -9591,7 +10181,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9613,12 +10203,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB41_1
+; GFX940-NEXT: s_cbranch_execnz .LBB45_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9634,7 +10224,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9662,12 +10252,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB41_1
+; GFX11-NEXT: s_cbranch_execnz .LBB45_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9680,7 +10270,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9701,12 +10291,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB41_1
+; GFX10-NEXT: s_cbranch_execnz .LBB45_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9721,7 +10311,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9742,12 +10332,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9762,7 +10352,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9781,12 +10371,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB41_1
+; GFX908-NEXT: s_cbranch_execnz .LBB45_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -9800,7 +10390,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9821,12 +10411,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB41_1
+; GFX8-NEXT: s_cbranch_execnz .LBB45_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -9840,7 +10430,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9858,12 +10448,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB41_1
+; GFX7-NEXT: s_cbranch_execnz .LBB45_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, bfloat %val seq_cst
+ %unused = atomicrmw fmin ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -9871,8 +10461,8 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; <2 x half>
; --------------------------------------------------------------------
-define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9882,7 +10472,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -9897,19 +10487,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB42_1
+; GFX12-NEXT: s_cbranch_execnz .LBB46_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -9923,19 +10513,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB42_1
+; GFX940-NEXT: s_cbranch_execnz .LBB46_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -9951,19 +10541,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB42_1
+; GFX11-NEXT: s_cbranch_execnz .LBB46_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -9977,19 +10567,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB42_1
+; GFX10-NEXT: s_cbranch_execnz .LBB46_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -10001,19 +10591,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -10025,20 +10615,20 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB42_1
+; GFX908-NEXT: s_cbranch_execnz .LBB46_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -10053,13 +10643,13 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB42_1
+; GFX8-NEXT: s_cbranch_execnz .LBB46_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2f16:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v5, v[0:1]
@@ -10072,7 +10662,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -10095,18 +10685,18 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16(ptr %ptr, <2 x half> %val) #
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB42_1
+; GFX7-NEXT: s_cbranch_execnz .LBB46_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10116,7 +10706,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -10131,19 +10721,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB43_1
+; GFX12-NEXT: s_cbranch_execnz .LBB47_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -10157,19 +10747,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB43_1
+; GFX940-NEXT: s_cbranch_execnz .LBB47_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -10185,13 +10775,13 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB43_1
+; GFX11-NEXT: s_cbranch_execnz .LBB47_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -10199,7 +10789,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -10213,18 +10803,18 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB43_1
+; GFX10-NEXT: s_cbranch_execnz .LBB47_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -10236,19 +10826,19 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -10260,13 +10850,13 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB43_1
+; GFX908-NEXT: s_cbranch_execnz .LBB47_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -10275,7 +10865,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -10290,12 +10880,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB43_1
+; GFX8-NEXT: s_cbranch_execnz .LBB47_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -10310,7 +10900,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -10333,17 +10923,17 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB43_1
+; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10353,7 +10943,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -10368,13 +10958,13 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: s_cbranch_execnz .LBB48_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v4, v0
@@ -10388,7 +10978,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -10402,12 +10992,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB44_1
+; GFX940-NEXT: s_cbranch_execnz .LBB48_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -10419,7 +11009,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX11-NEXT: flat_load_b32 v0, v[4:5]
; GFX11-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v0
@@ -10435,12 +11025,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: s_cbranch_execnz .LBB48_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -10448,7 +11038,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -10462,12 +11052,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB44_1
+; GFX10-NEXT: s_cbranch_execnz .LBB48_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -10477,7 +11067,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX90A-NEXT: flat_load_dword v0, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -10489,12 +11079,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -10504,7 +11094,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
@@ -10516,12 +11106,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB44_1
+; GFX908-NEXT: s_cbranch_execnz .LBB48_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -10530,7 +11120,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -10545,12 +11135,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB44_1
+; GFX8-NEXT: s_cbranch_execnz .LBB48_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -10565,7 +11155,7 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -10588,17 +11178,17 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB44_1
+; GFX7-NEXT: s_cbranch_execnz .LBB48_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 -512
- %result = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10608,7 +11198,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -10623,18 +11213,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB45_1
+; GFX12-NEXT: s_cbranch_execnz .LBB49_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10648,18 +11238,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB45_1
+; GFX940-NEXT: s_cbranch_execnz .LBB49_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10675,18 +11265,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB45_1
+; GFX11-NEXT: s_cbranch_execnz .LBB49_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10700,18 +11290,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB45_1
+; GFX10-NEXT: s_cbranch_execnz .LBB49_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10723,18 +11313,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10746,19 +11336,19 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB45_1
+; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -10773,12 +11363,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB45_1
+; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2f16:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v5, v[0:1]
@@ -10791,7 +11381,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -10814,16 +11404,16 @@ define void @flat_agent_atomic_fmin_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB45_1
+; GFX7-NEXT: s_cbranch_execnz .LBB49_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10833,7 +11423,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -10848,18 +11438,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-NEXT: s_cbranch_execnz .LBB50_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10873,18 +11463,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB46_1
+; GFX940-NEXT: s_cbranch_execnz .LBB50_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10900,12 +11490,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-NEXT: s_cbranch_execnz .LBB50_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -10913,7 +11503,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10927,18 +11517,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB46_1
+; GFX10-NEXT: s_cbranch_execnz .LBB50_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10950,18 +11540,18 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -10973,12 +11563,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB46_1
+; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -10987,7 +11577,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -11002,12 +11592,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB46_1
+; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -11022,7 +11612,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11045,17 +11635,17 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB46_1
+; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11065,7 +11655,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -11080,12 +11670,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -11097,7 +11687,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX940-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11111,12 +11701,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB47_1
+; GFX940-NEXT: s_cbranch_execnz .LBB51_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -11126,7 +11716,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX11-NEXT: flat_load_b32 v3, v[3:4]
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11142,12 +11732,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -11155,7 +11745,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11169,12 +11759,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB47_1
+; GFX10-NEXT: s_cbranch_execnz .LBB51_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -11184,7 +11774,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
@@ -11196,12 +11786,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -11211,7 +11801,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
@@ -11223,12 +11813,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB47_1
+; GFX908-NEXT: s_cbranch_execnz .LBB51_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -11237,7 +11827,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -11252,12 +11842,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB47_1
+; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -11272,7 +11862,7 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11295,17 +11885,17 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB47_1
+; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 -512
- %unused = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11315,7 +11905,7 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11330,19 +11920,19 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-NEXT: s_cbranch_execnz .LBB52_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11356,19 +11946,19 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB48_1
+; GFX940-NEXT: s_cbranch_execnz .LBB52_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11384,13 +11974,13 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-NEXT: s_cbranch_execnz .LBB52_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -11398,7 +11988,7 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -11412,18 +12002,18 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB48_1
+; GFX10-NEXT: s_cbranch_execnz .LBB52_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11437,19 +12027,19 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11461,13 +12051,13 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB48_1
+; GFX908-NEXT: s_cbranch_execnz .LBB52_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -11476,7 +12066,7 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -11491,12 +12081,12 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB48_1
+; GFX8-NEXT: s_cbranch_execnz .LBB52_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -11511,7 +12101,7 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -11534,17 +12124,17 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB48_1
+; GFX7-NEXT: s_cbranch_execnz .LBB52_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, <2 x half> %val seq_cst
+ %result = atomicrmw fmin ptr %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11554,7 +12144,7 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -11569,18 +12159,18 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB49_1
+; GFX12-NEXT: s_cbranch_execnz .LBB53_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11594,18 +12184,18 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB49_1
+; GFX940-NEXT: s_cbranch_execnz .LBB53_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11621,12 +12211,12 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB49_1
+; GFX11-NEXT: s_cbranch_execnz .LBB53_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -11634,7 +12224,7 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11648,18 +12238,18 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB49_1
+; GFX10-NEXT: s_cbranch_execnz .LBB53_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11673,18 +12263,18 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -11696,12 +12286,12 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB49_1
+; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -11710,7 +12300,7 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -11725,12 +12315,12 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB49_1
+; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -11745,7 +12335,7 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11768,12 +12358,12 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB49_1
+; GFX7-NEXT: s_cbranch_execnz .LBB53_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, <2 x half> %val seq_cst
+ %unused = atomicrmw fmin ptr %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -11781,8 +12371,8 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
; <2 x bfloat>
; --------------------------------------------------------------------
-define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11793,7 +12383,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -11825,13 +12415,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB50_1
+; GFX12-NEXT: s_cbranch_execnz .LBB54_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -11840,7 +12430,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -11867,13 +12457,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB50_1
+; GFX940-NEXT: s_cbranch_execnz .LBB54_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
@@ -11882,7 +12472,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -11915,21 +12505,21 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB50_1
+; GFX11-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -11956,13 +12546,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB50_1
+; GFX10-NEXT: s_cbranch_execnz .LBB54_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -11971,7 +12561,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -11996,13 +12586,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -12011,7 +12601,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -12036,20 +12626,20 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB50_1
+; GFX908-NEXT: s_cbranch_execnz .LBB54_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -12077,13 +12667,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB50_1
+; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2bf16:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v5, v[0:1]
@@ -12095,7 +12685,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -12115,18 +12705,18 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB50_1
+; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12137,7 +12727,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -12169,13 +12759,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB51_1
+; GFX12-NEXT: s_cbranch_execnz .LBB55_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -12184,7 +12774,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -12211,13 +12801,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB51_1
+; GFX940-NEXT: s_cbranch_execnz .LBB55_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -12226,7 +12816,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -12259,14 +12849,14 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB51_1
+; GFX11-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -12275,7 +12865,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -12302,12 +12892,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB51_1
+; GFX10-NEXT: s_cbranch_execnz .LBB55_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -12316,7 +12906,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -12341,13 +12931,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -12356,7 +12946,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -12381,13 +12971,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB51_1
+; GFX908-NEXT: s_cbranch_execnz .LBB55_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -12396,7 +12986,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -12424,12 +13014,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB51_1
+; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -12443,7 +13033,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -12463,17 +13053,17 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB51_1
+; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12484,7 +13074,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -12516,13 +13106,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB52_1
+; GFX12-NEXT: s_cbranch_execnz .LBB56_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v4, v0
@@ -12539,7 +13129,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v0
@@ -12566,12 +13156,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB52_1
+; GFX940-NEXT: s_cbranch_execnz .LBB56_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -12586,7 +13176,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v0
@@ -12619,13 +13209,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB52_1
+; GFX11-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -12634,7 +13224,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -12661,12 +13251,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB52_1
+; GFX10-NEXT: s_cbranch_execnz .LBB56_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -12679,7 +13269,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
@@ -12704,12 +13294,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -12722,7 +13312,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
@@ -12747,12 +13337,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB52_1
+; GFX908-NEXT: s_cbranch_execnz .LBB56_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -12761,7 +13351,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -12789,12 +13379,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB52_1
+; GFX8-NEXT: s_cbranch_execnz .LBB56_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -12808,7 +13398,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -12828,17 +13418,17 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB52_1
+; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 -512
- %result = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12849,7 +13439,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -12880,12 +13470,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-NEXT: s_cbranch_execnz .LBB57_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1]
@@ -12894,7 +13484,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -12921,12 +13511,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB53_1
+; GFX940-NEXT: s_cbranch_execnz .LBB57_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
@@ -12935,7 +13525,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -12967,20 +13557,20 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13007,12 +13597,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB53_1
+; GFX10-NEXT: s_cbranch_execnz .LBB57_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
@@ -13021,7 +13611,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13046,12 +13636,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
@@ -13060,7 +13650,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13085,19 +13675,19 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB53_1
+; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13125,12 +13715,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB53_1
+; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v4, v[0:1]
@@ -13142,7 +13732,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -13162,16 +13752,16 @@ define void @flat_agent_atomic_fmin_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB53_1
+; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13182,7 +13772,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13213,12 +13803,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-NEXT: s_cbranch_execnz .LBB58_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13227,7 +13817,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13254,12 +13844,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB54_1
+; GFX940-NEXT: s_cbranch_execnz .LBB58_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -13268,7 +13858,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13300,13 +13890,13 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -13315,7 +13905,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13342,12 +13932,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB54_1
+; GFX10-NEXT: s_cbranch_execnz .LBB58_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13356,7 +13946,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13381,12 +13971,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13395,7 +13985,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13420,12 +14010,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB54_1
+; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -13434,7 +14024,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13462,12 +14052,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB54_1
+; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -13481,7 +14071,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -13501,17 +14091,17 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB54_1
+; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13522,7 +14112,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13553,12 +14143,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-NEXT: s_cbranch_execnz .LBB59_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -13573,7 +14163,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13600,12 +14190,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB55_1
+; GFX940-NEXT: s_cbranch_execnz .LBB59_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -13618,7 +14208,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13650,13 +14240,13 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
@@ -13665,7 +14255,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13692,12 +14282,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB55_1
+; GFX10-NEXT: s_cbranch_execnz .LBB59_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -13710,7 +14300,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -13735,12 +14325,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -13753,7 +14343,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
@@ -13778,12 +14368,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB55_1
+; GFX908-NEXT: s_cbranch_execnz .LBB59_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -13792,7 +14382,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -13820,12 +14410,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB55_1
+; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
@@ -13839,7 +14429,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -13859,17 +14449,17 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB55_1
+; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 -512
- %unused = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13880,7 +14470,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -13912,13 +14502,13 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -13927,7 +14517,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -13954,13 +14544,13 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB56_1
+; GFX940-NEXT: s_cbranch_execnz .LBB60_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -13969,7 +14559,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -14002,14 +14592,14 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
@@ -14018,7 +14608,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
-; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
@@ -14045,12 +14635,12 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB56_1
+; GFX10-NEXT: s_cbranch_execnz .LBB60_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14059,7 +14649,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -14086,13 +14676,13 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14101,7 +14691,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -14126,13 +14716,13 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB56_1
+; GFX908-NEXT: s_cbranch_execnz .LBB60_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -14141,7 +14731,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -14169,12 +14759,12 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB56_1
+; GFX8-NEXT: s_cbranch_execnz .LBB60_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fc, v0
@@ -14188,7 +14778,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14208,17 +14798,17 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB56_1
+; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %result = atomicrmw fmin ptr %gep, <2 x bfloat> %val seq_cst
+ %result = atomicrmw fmin ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -14229,7 +14819,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14260,12 +14850,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-NEXT: s_cbranch_execnz .LBB61_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14274,7 +14864,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14301,12 +14891,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB57_1
+; GFX940-NEXT: s_cbranch_execnz .LBB61_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
@@ -14315,7 +14905,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14347,13 +14937,13 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
@@ -14362,7 +14952,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v3, v[0:1]
-; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14389,12 +14979,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB57_1
+; GFX10-NEXT: s_cbranch_execnz .LBB61_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14403,7 +14993,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14430,12 +15020,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
@@ -14444,7 +15034,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14469,12 +15059,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB57_1
+; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -14483,7 +15073,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14511,12 +15101,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB57_1
+; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
@@ -14530,7 +15120,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -14550,14 +15140,16 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB57_1
+; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr %ptr, i64 511
- %unused = atomicrmw fmin ptr %gep, <2 x bfloat> %val seq_cst
+ %unused = atomicrmw fmin ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index ae5dca4aa86fb..e7d62fdc00cff 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -13,8 +13,8 @@
; float
; --------------------------------------------------------------------
-define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f32:
+define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -27,7 +27,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -52,7 +52,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -62,7 +62,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -72,7 +72,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -96,7 +96,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -120,7 +120,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -144,7 +144,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -157,7 +157,7 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f32:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -170,12 +170,12 @@ define float @global_agent_atomic_fmax_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -188,7 +188,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -213,7 +213,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -223,7 +223,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -233,7 +233,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -257,7 +257,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -281,7 +281,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -306,7 +306,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -319,7 +319,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -333,12 +333,12 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -351,7 +351,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -376,7 +376,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -386,7 +386,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -396,7 +396,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -420,7 +420,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -444,7 +444,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -469,7 +469,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -482,7 +482,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -496,12 +496,12 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f32:
+define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -514,7 +514,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -538,7 +538,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -548,7 +548,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -558,7 +558,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -581,7 +581,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -604,7 +604,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -627,7 +627,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -639,7 +639,7 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f32:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -651,12 +651,12 @@ define void @global_agent_atomic_fmax_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -669,7 +669,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -693,7 +693,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -703,7 +703,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -713,7 +713,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -736,7 +736,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -759,7 +759,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -784,7 +784,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -796,7 +796,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -809,12 +809,12 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -827,7 +827,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -851,7 +851,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -861,7 +861,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -871,7 +871,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -894,7 +894,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -917,7 +917,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -942,7 +942,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -954,7 +954,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -967,12 +967,12 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+define float @global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1003,7 +1003,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1028,7 +1028,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -1056,7 +1056,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1082,7 +1082,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1108,7 +1108,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1132,7 +1132,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -1157,7 +1157,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -1187,7 +1187,7 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -1219,12 +1219,12 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+define void @global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1254,7 +1254,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1278,7 +1278,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -1305,7 +1305,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1330,7 +1330,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1355,7 +1355,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1378,7 +1378,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -1403,7 +1403,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -1432,7 +1432,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -1463,16 +1463,12 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-; --------------------------------------------------------------------
-; float with ftz/daz
-; --------------------------------------------------------------------
-
-define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1485,7 +1481,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -1510,7 +1506,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1520,7 +1516,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1530,7 +1526,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -1554,7 +1550,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -1578,7 +1574,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -1602,7 +1598,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -1615,7 +1611,7 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -1628,12 +1624,12 @@ define float @global_agent_atomic_fmax_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1641,15 +1637,15 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1659,7 +1655,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1671,30 +1667,30 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:2044 glc
+; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:2044 glc
+; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1703,7 +1699,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1715,10 +1711,10 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1727,7 +1723,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -1739,64 +1735,66 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; --------------------------------------------------------------------
+; float with ftz/daz
+; --------------------------------------------------------------------
+
+define float @global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1804,15 +1802,15 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
@@ -1822,7 +1820,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1834,30 +1832,30 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:-2048 glc
+; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:-2048 glc
+; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
@@ -1866,7 +1864,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1878,10 +1876,10 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
@@ -1890,7 +1888,7 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -1902,64 +1900,62 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @global_agent_atomic_fmax_noret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1967,154 +1963,162 @@ define void @global_agent_atomic_fmax_noret_f32__ftz(ptr addrspace(1) %ptr, floa
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off
-; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB11_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:2044 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
+ %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2122,157 +2126,162 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspac
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:2044
-; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB12_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:2044
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:-2048 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:2044
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:-2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
+ %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+define void @global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2280,15 +2289,15 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:-2048
+; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2297,7 +2306,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2309,30 +2318,30 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:-2048
+; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:-2048
+; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2340,7 +2349,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2352,10 +2361,10 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2363,7 +2372,7 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2375,11 +2384,9 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
@@ -2400,322 +2407,210 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @global_system_atomic_fmax_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:2044
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v5, v3
-; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: buffer_inv sc0 sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB14_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:2044
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
-; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:2044
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_1
-; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; GFX7-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v3, v4
-; GFX7-NEXT: v_mov_b32_e32 v4, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; GFX6-NEXT: v_max_f32_e32 v4, v3, v2
-; GFX6-NEXT: v_mov_b32_e32 v3, v4
-; GFX6-NEXT: v_mov_b32_e32 v4, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_1
-; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst
- ret float %result
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:-2048
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
@@ -2723,10 +2618,10 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
@@ -2736,62 +2631,30 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:-2048
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
-; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:-2048
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB15_1
-; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
@@ -2799,10 +2662,8 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2813,10 +2674,10 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
@@ -2824,7 +2685,7 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2836,11 +2697,11 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
@@ -2861,7 +2722,222 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define float @global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
+; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX940-NEXT: buffer_wbl2 sc0 sc1
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB16_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB16_1
+; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_mov_b32_e32 v0, v3
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -2870,77 +2946,666 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX7-NEXT: v_mov_b32_e32 v6, v3
-; GFX7-NEXT: v_mov_b32_e32 v5, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX7-NEXT: v_mov_b32_e32 v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_cbranch_execnz .LBB16_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v0, v3
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v5, v3
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v5
+; GFX6-NEXT: v_max_f32_e32 v4, v3, v2
+; GFX6-NEXT: v_mov_b32_e32 v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_cbranch_execnz .LBB16_1
+; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v0, v3
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
+ %result = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
+}
+
+define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc0 sc1
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB17_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB17_1
+; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX7-NEXT: v_mov_b32_e32 v6, v3
+; GFX7-NEXT: v_mov_b32_e32 v5, v2
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v3, v5
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX6-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v6, v3
+; GFX6-NEXT: v_mov_b32_e32 v5, v2
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v3, v5
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_cbranch_execnz .LBB17_1
+; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+; --------------------------------------------------------------------
+; double
+; --------------------------------------------------------------------
+
+define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:2040 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off offset:2040 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:2040 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB19_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v3, v5
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB15_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX6-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v6, v3
-; GFX6-NEXT: v_mov_b32_e32 v5, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v5
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB15_1
-; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst
- ret void
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
+ %result = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
}
-; --------------------------------------------------------------------
-; double
-; --------------------------------------------------------------------
-
-define double @global_agent_atomic_fmax_ret_f64(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f64:
+define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
@@ -2948,35 +3613,35 @@ define double @global_agent_atomic_fmax_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: s_cbranch_execnz .LBB20_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
@@ -2984,7 +3649,7 @@ define double @global_agent_atomic_fmax_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -2992,89 +3657,89 @@ define double @global_agent_atomic_fmax_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: s_cbranch_execnz .LBB20_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
@@ -3082,13 +3747,13 @@ define double @global_agent_atomic_fmax_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f64:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
@@ -3096,373 +3761,355 @@ define double @global_agent_atomic_fmax_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
+ %result = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret double %result
}
-define double @global_agent_atomic_fmax_ret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:2040 sc0
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off offset:2040 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:2040 glc
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_mov_b32_e32 v0, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_mov_b32_e32 v0, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
- %result = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define double @global_agent_atomic_fmax_ret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: s_cbranch_execnz .LBB22_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 sc0
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:2040
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: s_cbranch_execnz .LBB22_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off offset:2040
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:2040
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_mov_b32_e32 v0, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_mov_b32_e32 v0, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
- %result = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @global_agent_atomic_fmax_noret_f64(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f64:
+define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
@@ -3470,34 +4117,34 @@ define void @global_agent_atomic_fmax_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: s_cbranch_execnz .LBB23_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:-2048
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -3506,41 +4153,41 @@ define void @global_agent_atomic_fmax_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: s_cbranch_execnz .LBB23_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off offset:-2048
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:-2048
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
@@ -3548,18 +4195,20 @@ define void @global_agent_atomic_fmax_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB19_1
+; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
@@ -3572,373 +4221,388 @@ define void @global_agent_atomic_fmax_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f64:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB20_1
+; GFX12-NEXT: s_cbranch_execnz .LBB24_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:2040
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB20_1
+; GFX11-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off offset:2040
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:2040
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB20_1
+; GFX908-NEXT: s_cbranch_execnz .LBB24_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB20_1
+; GFX8-NEXT: s_cbranch_execnz .LBB24_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
- %unused = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret double %result
}
-define void @global_agent_atomic_fmax_noret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB21_1
+; GFX12-NEXT: s_cbranch_execnz .LBB25_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:-2048
+; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off offset:-2048
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:-2048
+; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_1
+; GFX908-NEXT: s_cbranch_execnz .LBB25_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_1
+; GFX8-NEXT: s_cbranch_execnz .LBB25_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
-; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
-; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
- %unused = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret double %result
}
; --------------------------------------------------------------------
; half
; --------------------------------------------------------------------
-define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f16:
+define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3956,7 +4620,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -3977,13 +4641,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB22_1
+; GFX12-NEXT: s_cbranch_execnz .LBB26_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -3996,7 +4660,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4012,13 +4676,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB22_1
+; GFX940-NEXT: s_cbranch_execnz .LBB26_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -4032,7 +4696,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4054,13 +4718,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -4072,7 +4736,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4089,13 +4753,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB22_1
+; GFX10-NEXT: s_cbranch_execnz .LBB26_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -4108,7 +4772,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4123,13 +4787,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -4142,7 +4806,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4157,13 +4821,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB22_1
+; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -4176,7 +4840,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4192,13 +4856,13 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB22_1
+; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -4215,7 +4879,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
; GFX7-NEXT: v_not_b32_e32 v7, v2
-; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -4234,14 +4898,14 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB22_1
+; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f16:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -4258,7 +4922,7 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
; GFX6-NEXT: v_not_b32_e32 v7, v2
-; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -4277,19 +4941,19 @@ define half @global_agent_atomic_fmax_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB22_1
+; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4308,7 +4972,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -4329,13 +4993,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB23_1
+; GFX12-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -4350,7 +5014,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4366,13 +5030,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB23_1
+; GFX940-NEXT: s_cbranch_execnz .LBB27_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4387,7 +5051,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4409,13 +5073,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4428,7 +5092,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4445,13 +5109,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB23_1
+; GFX10-NEXT: s_cbranch_execnz .LBB27_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4465,7 +5129,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4480,13 +5144,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4500,7 +5164,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4515,13 +5179,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB23_1
+; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -4535,7 +5199,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4551,13 +5215,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB23_1
+; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -4575,7 +5239,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX7-NEXT: v_not_b32_e32 v8, v2
-; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -4594,14 +5258,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB23_1
+; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -4619,7 +5283,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX6-NEXT: v_not_b32_e32 v8, v2
-; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -4639,7 +5303,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB23_1
+; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -4647,12 +5311,12 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4671,7 +5335,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -4692,13 +5356,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB24_1
+; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -4714,7 +5378,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4730,13 +5394,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB24_1
+; GFX940-NEXT: s_cbranch_execnz .LBB28_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4751,7 +5415,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4773,13 +5437,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4792,7 +5456,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4809,13 +5473,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB24_1
+; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4829,7 +5493,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4844,13 +5508,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4864,7 +5528,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4879,13 +5543,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_1
+; GFX908-NEXT: s_cbranch_execnz .LBB28_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4899,7 +5563,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4915,13 +5579,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB24_1
+; GFX8-NEXT: s_cbranch_execnz .LBB28_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -4939,7 +5603,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX7-NEXT: v_not_b32_e32 v8, v2
-; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -4958,14 +5622,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB24_1
+; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -4983,7 +5647,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX6-NEXT: v_not_b32_e32 v8, v2
-; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -5003,7 +5667,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB24_1
+; GFX6-NEXT: s_cbranch_execnz .LBB28_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -5011,12 +5675,12 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
- %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f16:
+define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5034,7 +5698,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5055,12 +5719,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB25_1
+; GFX12-NEXT: s_cbranch_execnz .LBB29_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -5073,7 +5737,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX940-NEXT: v_not_b32_e32 v6, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -5089,12 +5753,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB25_1
+; GFX940-NEXT: s_cbranch_execnz .LBB29_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -5108,7 +5772,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5130,12 +5794,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-NEXT: s_cbranch_execnz .LBB29_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -5147,7 +5811,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5164,12 +5828,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB25_1
+; GFX10-NEXT: s_cbranch_execnz .LBB29_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -5182,7 +5846,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -5197,12 +5861,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -5215,7 +5879,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5230,12 +5894,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_1
+; GFX908-NEXT: s_cbranch_execnz .LBB29_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -5248,7 +5912,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5264,12 +5928,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB25_1
+; GFX8-NEXT: s_cbranch_execnz .LBB29_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -5286,7 +5950,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
@@ -5305,12 +5969,12 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB25_1
+; GFX7-NEXT: s_cbranch_execnz .LBB29_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f16:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -5327,7 +5991,7 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
@@ -5347,17 +6011,17 @@ define void @global_agent_atomic_fmax_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB25_1
+; GFX6-NEXT: s_cbranch_execnz .LBB29_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5376,7 +6040,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5397,12 +6061,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB26_1
+; GFX12-NEXT: s_cbranch_execnz .LBB30_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -5417,7 +6081,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5433,12 +6097,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB26_1
+; GFX940-NEXT: s_cbranch_execnz .LBB30_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -5453,7 +6117,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5475,12 +6139,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB26_1
+; GFX11-NEXT: s_cbranch_execnz .LBB30_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -5493,7 +6157,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5510,12 +6174,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB26_1
+; GFX10-NEXT: s_cbranch_execnz .LBB30_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5529,7 +6193,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5544,12 +6208,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5563,7 +6227,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5578,12 +6242,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB26_1
+; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -5597,7 +6261,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5613,12 +6277,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB26_1
+; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -5636,7 +6300,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5655,12 +6319,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB26_1
+; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -5678,7 +6342,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX6-NEXT: v_not_b32_e32 v6, v2
-; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5698,18 +6362,18 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB26_1
+; GFX6-NEXT: s_cbranch_execnz .LBB30_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5728,7 +6392,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5749,12 +6413,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB27_1
+; GFX12-NEXT: s_cbranch_execnz .LBB31_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -5770,7 +6434,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5786,12 +6450,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB27_1
+; GFX940-NEXT: s_cbranch_execnz .LBB31_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5806,7 +6470,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5828,12 +6492,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB27_1
+; GFX11-NEXT: s_cbranch_execnz .LBB31_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5846,7 +6510,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5863,12 +6527,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB27_1
+; GFX10-NEXT: s_cbranch_execnz .LBB31_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5882,7 +6546,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5897,12 +6561,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5916,7 +6580,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5931,12 +6595,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB27_1
+; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5950,7 +6614,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5966,12 +6630,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB27_1
+; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -5989,7 +6653,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6008,12 +6672,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB27_1
+; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -6031,7 +6695,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX6-NEXT: v_not_b32_e32 v6, v2
-; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6051,18 +6715,18 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB27_1
+; GFX6-NEXT: s_cbranch_execnz .LBB31_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
- %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6072,7 +6736,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -6090,20 +6754,20 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB28_1
+; GFX12-NEXT: s_cbranch_execnz .LBB32_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -6117,19 +6781,19 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB28_1
+; GFX940-NEXT: s_cbranch_execnz .LBB32_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -6148,19 +6812,19 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB28_1
+; GFX11-NEXT: s_cbranch_execnz .LBB32_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -6176,20 +6840,20 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB28_1
+; GFX10-NEXT: s_cbranch_execnz .LBB32_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -6202,20 +6866,20 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -6228,13 +6892,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB28_1
+; GFX908-NEXT: s_cbranch_execnz .LBB32_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -6242,7 +6906,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -6256,12 +6920,12 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB28_1
+; GFX8-NEXT: s_cbranch_execnz .LBB32_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -6272,7 +6936,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
-; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6289,13 +6953,13 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB28_1
+; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -6306,7 +6970,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
-; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6324,19 +6988,19 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB28_1
+; GFX6-NEXT: s_cbranch_execnz .LBB32_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6346,7 +7010,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f16_e32 v2, v3, v3
@@ -6364,19 +7028,19 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB29_1
+; GFX12-NEXT: s_cbranch_execnz .LBB33_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v4, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6390,18 +7054,18 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB29_1
+; GFX940-NEXT: s_cbranch_execnz .LBB33_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6420,18 +7084,18 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB29_1
+; GFX11-NEXT: s_cbranch_execnz .LBB33_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6447,19 +7111,19 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB29_1
+; GFX10-NEXT: s_cbranch_execnz .LBB33_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6472,19 +7136,19 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6497,12 +7161,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB29_1
+; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -6510,7 +7174,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
-; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6524,12 +7188,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB29_1
+; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -6540,7 +7204,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6557,12 +7221,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB29_1
+; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -6573,7 +7237,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6591,18 +7255,18 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos(ptr addrs
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB29_1
+; GFX6-NEXT: s_cbranch_execnz .LBB33_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6621,7 +7285,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -6642,13 +7306,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB30_1
+; GFX12-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -6663,7 +7327,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6679,13 +7343,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB30_1
+; GFX940-NEXT: s_cbranch_execnz .LBB34_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6700,7 +7364,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -6722,13 +7386,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB30_1
+; GFX11-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6741,7 +7405,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -6758,13 +7422,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB30_1
+; GFX10-NEXT: s_cbranch_execnz .LBB34_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6778,7 +7442,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -6795,13 +7459,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6815,7 +7479,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -6830,13 +7494,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_1
+; GFX908-NEXT: s_cbranch_execnz .LBB34_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -6850,7 +7514,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -6866,13 +7530,13 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB30_1
+; GFX8-NEXT: s_cbranch_execnz .LBB34_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -6890,7 +7554,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX7-NEXT: v_not_b32_e32 v8, v2
-; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -6909,14 +7573,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB30_1
+; GFX7-NEXT: s_cbranch_execnz .LBB34_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -6934,7 +7598,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX6-NEXT: v_not_b32_e32 v8, v2
-; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -6954,7 +7618,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB30_1
+; GFX6-NEXT: s_cbranch_execnz .LBB34_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -6962,12 +7626,12 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmax ptr addrspace(1) %gep, half %val seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6986,7 +7650,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7007,12 +7671,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB31_1
+; GFX12-NEXT: s_cbranch_execnz .LBB35_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -7027,7 +7691,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7043,12 +7707,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB31_1
+; GFX940-NEXT: s_cbranch_execnz .LBB35_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -7063,7 +7727,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7085,12 +7749,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB31_1
+; GFX11-NEXT: s_cbranch_execnz .LBB35_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -7103,7 +7767,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7120,12 +7784,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB31_1
+; GFX10-NEXT: s_cbranch_execnz .LBB35_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -7139,7 +7803,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7156,12 +7820,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -7175,7 +7839,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7190,12 +7854,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB31_1
+; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -7209,7 +7873,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7225,12 +7889,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB31_1
+; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -7248,7 +7912,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7267,12 +7931,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB31_1
+; GFX7-NEXT: s_cbranch_execnz .LBB35_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -7290,7 +7954,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX6-NEXT: v_not_b32_e32 v6, v2
-; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7310,13 +7974,13 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB31_1
+; GFX6-NEXT: s_cbranch_execnz .LBB35_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -7324,8 +7988,8 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos(ptr addrspace(1)
; bfloat
; --------------------------------------------------------------------
-define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16:
+define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7342,7 +8006,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -7370,13 +8034,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -7390,7 +8054,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -7412,13 +8076,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB32_1
+; GFX940-NEXT: s_cbranch_execnz .LBB36_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -7432,7 +8096,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7461,13 +8125,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -7479,7 +8143,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7500,13 +8164,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB32_1
+; GFX10-NEXT: s_cbranch_execnz .LBB36_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -7520,7 +8184,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7539,13 +8203,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -7559,7 +8223,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7578,13 +8242,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB32_1
+; GFX908-NEXT: s_cbranch_execnz .LBB36_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -7597,7 +8261,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -7618,13 +8282,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB32_1
+; GFX8-NEXT: s_cbranch_execnz .LBB36_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -7641,7 +8305,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -7661,14 +8325,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB32_1
+; GFX7-NEXT: s_cbranch_execnz .LBB36_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -7685,7 +8349,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -7705,19 +8369,19 @@ define bfloat @global_agent_atomic_fmax_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB32_1
+; GFX6-NEXT: s_cbranch_execnz .LBB36_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7736,7 +8400,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -7764,13 +8428,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -7786,7 +8450,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -7808,13 +8472,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB33_1
+; GFX940-NEXT: s_cbranch_execnz .LBB37_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7830,7 +8494,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7859,13 +8523,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7878,7 +8542,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7899,13 +8563,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB33_1
+; GFX10-NEXT: s_cbranch_execnz .LBB37_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7920,7 +8584,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7939,13 +8603,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7960,7 +8624,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7979,13 +8643,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB33_1
+; GFX908-NEXT: s_cbranch_execnz .LBB37_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -7999,7 +8663,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -8020,13 +8684,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB33_1
+; GFX8-NEXT: s_cbranch_execnz .LBB37_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -8044,7 +8708,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8064,14 +8728,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB33_1
+; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -8089,7 +8753,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8110,7 +8774,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB33_1
+; GFX6-NEXT: s_cbranch_execnz .LBB37_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -8118,12 +8782,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8142,7 +8806,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -8170,13 +8834,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -8193,7 +8857,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -8215,13 +8879,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB34_1
+; GFX940-NEXT: s_cbranch_execnz .LBB38_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -8237,7 +8901,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -8266,13 +8930,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -8285,7 +8949,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -8306,13 +8970,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB34_1
+; GFX10-NEXT: s_cbranch_execnz .LBB38_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -8327,7 +8991,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -8346,13 +9010,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -8367,7 +9031,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -8386,13 +9050,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB34_1
+; GFX908-NEXT: s_cbranch_execnz .LBB38_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -8406,7 +9070,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -8427,13 +9091,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB34_1
+; GFX8-NEXT: s_cbranch_execnz .LBB38_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -8451,7 +9115,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8471,14 +9135,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB34_1
+; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -8496,7 +9160,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8517,7 +9181,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB34_1
+; GFX6-NEXT: s_cbranch_execnz .LBB38_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -8525,12 +9189,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
- %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16:
+define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8547,7 +9211,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8574,12 +9238,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -8593,7 +9257,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8615,12 +9279,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB35_1
+; GFX940-NEXT: s_cbranch_execnz .LBB39_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -8634,7 +9298,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8662,12 +9326,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -8679,7 +9343,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8700,12 +9364,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB35_1
+; GFX10-NEXT: s_cbranch_execnz .LBB39_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -8719,7 +9383,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8738,12 +9402,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -8757,7 +9421,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8776,12 +9440,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB35_1
+; GFX908-NEXT: s_cbranch_execnz .LBB39_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -8794,7 +9458,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8815,12 +9479,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB35_1
+; GFX8-NEXT: s_cbranch_execnz .LBB39_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -8837,7 +9501,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8857,12 +9521,12 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB35_1
+; GFX7-NEXT: s_cbranch_execnz .LBB39_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -8879,7 +9543,7 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8900,17 +9564,17 @@ define void @global_agent_atomic_fmax_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB35_1
+; GFX6-NEXT: s_cbranch_execnz .LBB39_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8929,7 +9593,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8956,12 +9620,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: s_cbranch_execnz .LBB40_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -8977,7 +9641,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8999,12 +9663,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB36_1
+; GFX940-NEXT: s_cbranch_execnz .LBB40_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9020,7 +9684,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9048,12 +9712,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: s_cbranch_execnz .LBB40_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9066,7 +9730,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9087,12 +9751,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB36_1
+; GFX10-NEXT: s_cbranch_execnz .LBB40_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9107,7 +9771,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9126,12 +9790,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9146,7 +9810,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9165,12 +9829,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB36_1
+; GFX908-NEXT: s_cbranch_execnz .LBB40_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -9184,7 +9848,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9205,12 +9869,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB36_1
+; GFX8-NEXT: s_cbranch_execnz .LBB40_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -9228,7 +9892,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9248,12 +9912,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB36_1
+; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -9271,7 +9935,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9292,18 +9956,18 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB36_1
+; GFX6-NEXT: s_cbranch_execnz .LBB40_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9322,7 +9986,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9349,12 +10013,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: s_cbranch_execnz .LBB41_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -9371,7 +10035,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9393,12 +10057,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB37_1
+; GFX940-NEXT: s_cbranch_execnz .LBB41_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -9414,7 +10078,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9442,12 +10106,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: s_cbranch_execnz .LBB41_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -9460,7 +10124,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9481,12 +10145,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB37_1
+; GFX10-NEXT: s_cbranch_execnz .LBB41_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -9501,7 +10165,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9520,12 +10184,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -9540,7 +10204,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9559,12 +10223,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB37_1
+; GFX908-NEXT: s_cbranch_execnz .LBB41_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -9578,7 +10242,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9599,12 +10263,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB37_1
+; GFX8-NEXT: s_cbranch_execnz .LBB41_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -9622,7 +10286,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9642,12 +10306,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB37_1
+; GFX7-NEXT: s_cbranch_execnz .LBB41_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -9665,7 +10329,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9686,18 +10350,18 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB37_1
+; GFX6-NEXT: s_cbranch_execnz .LBB41_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
- %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9707,7 +10371,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -9732,13 +10396,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -9746,7 +10410,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -9767,20 +10431,20 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB38_1
+; GFX940-NEXT: s_cbranch_execnz .LBB42_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -9806,19 +10470,19 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -9839,13 +10503,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB38_1
+; GFX10-NEXT: s_cbranch_execnz .LBB42_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -9853,7 +10517,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -9872,13 +10536,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -9886,7 +10550,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -9905,13 +10569,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB38_1
+; GFX908-NEXT: s_cbranch_execnz .LBB42_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -9919,7 +10583,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -9939,12 +10603,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB38_1
+; GFX8-NEXT: s_cbranch_execnz .LBB42_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -9955,7 +10619,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9973,13 +10637,13 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB38_1
+; GFX7-NEXT: s_cbranch_execnz .LBB42_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -9990,7 +10654,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10009,19 +10673,19 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB38_1
+; GFX6-NEXT: s_cbranch_execnz .LBB42_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10031,7 +10695,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10055,12 +10719,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: s_cbranch_execnz .LBB43_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -10068,7 +10732,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10089,19 +10753,19 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB39_1
+; GFX940-NEXT: s_cbranch_execnz .LBB43_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10126,18 +10790,18 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: s_cbranch_execnz .LBB43_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10158,12 +10822,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB39_1
+; GFX10-NEXT: s_cbranch_execnz .LBB43_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -10171,7 +10835,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10190,12 +10854,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -10203,7 +10867,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10222,12 +10886,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB39_1
+; GFX908-NEXT: s_cbranch_execnz .LBB43_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -10235,7 +10899,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10255,12 +10919,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB39_1
+; GFX8-NEXT: s_cbranch_execnz .LBB43_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -10271,7 +10935,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10289,12 +10953,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB39_1
+; GFX7-NEXT: s_cbranch_execnz .LBB43_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -10305,7 +10969,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10324,18 +10988,18 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB39_1
+; GFX6-NEXT: s_cbranch_execnz .LBB43_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10354,7 +11018,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -10382,13 +11046,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -10404,7 +11068,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -10426,13 +11090,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB40_1
+; GFX940-NEXT: s_cbranch_execnz .LBB44_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -10448,7 +11112,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -10477,13 +11141,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -10496,7 +11160,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -10517,13 +11181,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB40_1
+; GFX10-NEXT: s_cbranch_execnz .LBB44_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -10538,7 +11202,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -10559,13 +11223,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -10580,7 +11244,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -10599,13 +11263,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB40_1
+; GFX908-NEXT: s_cbranch_execnz .LBB44_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -10619,7 +11283,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -10640,13 +11304,13 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB40_1
+; GFX8-NEXT: s_cbranch_execnz .LBB44_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -10664,7 +11328,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -10684,14 +11348,14 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB40_1
+; GFX7-NEXT: s_cbranch_execnz .LBB44_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -10709,7 +11373,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -10730,7 +11394,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB40_1
+; GFX6-NEXT: s_cbranch_execnz .LBB44_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -10738,12 +11402,12 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10762,7 +11426,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -10789,12 +11453,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB41_1
+; GFX12-NEXT: s_cbranch_execnz .LBB45_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -10810,7 +11474,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10832,12 +11496,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB41_1
+; GFX940-NEXT: s_cbranch_execnz .LBB45_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -10853,7 +11517,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -10881,12 +11545,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB41_1
+; GFX11-NEXT: s_cbranch_execnz .LBB45_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -10899,7 +11563,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10920,12 +11584,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB41_1
+; GFX10-NEXT: s_cbranch_execnz .LBB45_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -10940,7 +11604,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10961,12 +11625,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -10981,7 +11645,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -11000,12 +11664,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB41_1
+; GFX908-NEXT: s_cbranch_execnz .LBB45_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -11019,7 +11683,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -11040,12 +11704,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB41_1
+; GFX8-NEXT: s_cbranch_execnz .LBB45_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -11063,7 +11727,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -11083,12 +11747,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB41_1
+; GFX7-NEXT: s_cbranch_execnz .LBB45_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -11106,7 +11770,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -11127,13 +11791,13 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB41_1
+; GFX6-NEXT: s_cbranch_execnz .LBB45_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -11141,8 +11805,8 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos(ptr addrspace(1
; <2 x half>
; --------------------------------------------------------------------
-define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16:
+define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11152,7 +11816,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11167,19 +11831,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB42_1
+; GFX12-NEXT: s_cbranch_execnz .LBB46_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11193,19 +11857,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB42_1
+; GFX940-NEXT: s_cbranch_execnz .LBB46_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11221,19 +11885,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB42_1
+; GFX11-NEXT: s_cbranch_execnz .LBB46_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -11247,19 +11911,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB42_1
+; GFX10-NEXT: s_cbranch_execnz .LBB46_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11271,19 +11935,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11295,20 +11959,20 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB42_1
+; GFX908-NEXT: s_cbranch_execnz .LBB46_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -11323,13 +11987,13 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB42_1
+; GFX8-NEXT: s_cbranch_execnz .LBB46_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -11346,7 +12010,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11371,14 +12035,14 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB42_1
+; GFX7-NEXT: s_cbranch_execnz .LBB46_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -11395,7 +12059,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11421,19 +12085,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB42_1
+; GFX6-NEXT: s_cbranch_execnz .LBB46_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11443,7 +12107,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11458,19 +12122,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB43_1
+; GFX12-NEXT: s_cbranch_execnz .LBB47_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11484,19 +12148,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB43_1
+; GFX940-NEXT: s_cbranch_execnz .LBB47_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11512,19 +12176,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB43_1
+; GFX11-NEXT: s_cbranch_execnz .LBB47_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -11538,19 +12202,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB43_1
+; GFX10-NEXT: s_cbranch_execnz .LBB47_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11562,19 +12226,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11586,13 +12250,13 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB43_1
+; GFX908-NEXT: s_cbranch_execnz .LBB47_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -11601,7 +12265,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -11616,12 +12280,12 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB43_1
+; GFX8-NEXT: s_cbranch_execnz .LBB47_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -11638,7 +12302,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11663,14 +12327,14 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB43_1
+; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -11687,7 +12351,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11713,7 +12377,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB43_1
+; GFX6-NEXT: s_cbranch_execnz .LBB47_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
@@ -11721,12 +12385,12 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspa
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11736,7 +12400,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11751,19 +12415,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: s_cbranch_execnz .LBB48_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11777,19 +12441,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB44_1
+; GFX940-NEXT: s_cbranch_execnz .LBB48_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11805,19 +12469,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: s_cbranch_execnz .LBB48_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -11831,19 +12495,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB44_1
+; GFX10-NEXT: s_cbranch_execnz .LBB48_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11855,19 +12519,19 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11879,13 +12543,13 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB44_1
+; GFX908-NEXT: s_cbranch_execnz .LBB48_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -11894,7 +12558,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -11909,12 +12573,12 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB44_1
+; GFX8-NEXT: s_cbranch_execnz .LBB48_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -11935,7 +12599,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -11960,12 +12624,12 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB44_1
+; GFX7-NEXT: s_cbranch_execnz .LBB48_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -11986,7 +12650,7 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -12012,18 +12676,18 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg(ptr addrspa
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB44_1
+; GFX6-NEXT: s_cbranch_execnz .LBB48_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16:
+define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12033,7 +12697,7 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -12048,18 +12712,18 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB45_1
+; GFX12-NEXT: s_cbranch_execnz .LBB49_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12073,18 +12737,18 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB45_1
+; GFX940-NEXT: s_cbranch_execnz .LBB49_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12100,18 +12764,18 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB45_1
+; GFX11-NEXT: s_cbranch_execnz .LBB49_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12125,18 +12789,18 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB45_1
+; GFX10-NEXT: s_cbranch_execnz .LBB49_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12148,18 +12812,18 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12171,19 +12835,19 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB45_1
+; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -12198,12 +12862,12 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB45_1
+; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -12220,7 +12884,7 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12245,12 +12909,12 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB45_1
+; GFX7-NEXT: s_cbranch_execnz .LBB49_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -12267,7 +12931,7 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12293,17 +12957,17 @@ define void @global_agent_atomic_fmax_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB45_1
+; GFX6-NEXT: s_cbranch_execnz .LBB49_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12313,7 +12977,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -12328,18 +12992,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-NEXT: s_cbranch_execnz .LBB50_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12353,18 +13017,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB46_1
+; GFX940-NEXT: s_cbranch_execnz .LBB50_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12380,18 +13044,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-NEXT: s_cbranch_execnz .LBB50_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12405,18 +13069,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB46_1
+; GFX10-NEXT: s_cbranch_execnz .LBB50_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12428,18 +13092,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12451,12 +13115,12 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB46_1
+; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -12465,7 +13129,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -12480,12 +13144,12 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB46_1
+; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -12502,7 +13166,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12527,12 +13191,12 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB46_1
+; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -12549,7 +13213,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12575,18 +13239,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB46_1
+; GFX6-NEXT: s_cbranch_execnz .LBB50_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12596,7 +13260,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -12611,18 +13275,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12636,18 +13300,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB47_1
+; GFX940-NEXT: s_cbranch_execnz .LBB51_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12663,18 +13327,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12688,18 +13352,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB47_1
+; GFX10-NEXT: s_cbranch_execnz .LBB51_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12711,18 +13375,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12734,12 +13398,12 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB47_1
+; GFX908-NEXT: s_cbranch_execnz .LBB51_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -12748,7 +13412,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -12763,12 +13427,12 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB47_1
+; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -12789,7 +13453,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12814,12 +13478,12 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB47_1
+; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -12840,7 +13504,7 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12866,18 +13530,18 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB47_1
+; GFX6-NEXT: s_cbranch_execnz .LBB51_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12887,7 +13551,7 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -12902,19 +13566,19 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-NEXT: s_cbranch_execnz .LBB52_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -12928,19 +13592,19 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB48_1
+; GFX940-NEXT: s_cbranch_execnz .LBB52_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -12956,19 +13620,19 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-NEXT: s_cbranch_execnz .LBB52_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -12982,19 +13646,19 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB48_1
+; GFX10-NEXT: s_cbranch_execnz .LBB52_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -13008,19 +13672,19 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -13032,13 +13696,13 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB48_1
+; GFX908-NEXT: s_cbranch_execnz .LBB52_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -13047,7 +13711,7 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -13062,12 +13726,12 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB48_1
+; GFX8-NEXT: s_cbranch_execnz .LBB52_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -13084,7 +13748,7 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -13109,14 +13773,14 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB48_1
+; GFX7-NEXT: s_cbranch_execnz .LBB52_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -13133,7 +13797,7 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -13159,7 +13823,7 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB48_1
+; GFX6-NEXT: s_cbranch_execnz .LBB52_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
@@ -13167,12 +13831,12 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos(ptr addrsp
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13182,7 +13846,7 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -13197,18 +13861,18 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB49_1
+; GFX12-NEXT: s_cbranch_execnz .LBB53_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13222,18 +13886,18 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB49_1
+; GFX940-NEXT: s_cbranch_execnz .LBB53_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13249,18 +13913,18 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB49_1
+; GFX11-NEXT: s_cbranch_execnz .LBB53_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13274,18 +13938,18 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB49_1
+; GFX10-NEXT: s_cbranch_execnz .LBB53_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13299,18 +13963,18 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13322,12 +13986,12 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB49_1
+; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -13336,7 +14000,7 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -13351,12 +14015,12 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB49_1
+; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -13373,7 +14037,7 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -13398,12 +14062,12 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB49_1
+; GFX7-NEXT: s_cbranch_execnz .LBB53_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -13420,7 +14084,7 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -13446,13 +14110,13 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB49_1
+; GFX6-NEXT: s_cbranch_execnz .LBB53_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -13460,8 +14124,8 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos(ptr addrspace(
; <2 x bfloat>
; --------------------------------------------------------------------
-define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13472,7 +14136,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -13504,13 +14168,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB50_1
+; GFX12-NEXT: s_cbranch_execnz .LBB54_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -13519,7 +14183,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -13546,13 +14210,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB50_1
+; GFX940-NEXT: s_cbranch_execnz .LBB54_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
@@ -13561,7 +14225,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -13594,21 +14258,21 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB50_1
+; GFX11-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -13635,13 +14299,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB50_1
+; GFX10-NEXT: s_cbranch_execnz .LBB54_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -13650,7 +14314,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -13675,13 +14339,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -13690,7 +14354,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -13715,20 +14379,20 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB50_1
+; GFX908-NEXT: s_cbranch_execnz .LBB54_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -13756,13 +14420,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB50_1
+; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -13778,7 +14442,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -13800,14 +14464,14 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB50_1
+; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -13823,7 +14487,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -13846,19 +14510,19 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB50_1
+; GFX6-NEXT: s_cbranch_execnz .LBB54_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v3
; GFX6-NEXT: v_mov_b32_e32 v1, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13869,7 +14533,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -13901,13 +14565,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB51_1
+; GFX12-NEXT: s_cbranch_execnz .LBB55_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -13916,7 +14580,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -13943,13 +14607,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB51_1
+; GFX940-NEXT: s_cbranch_execnz .LBB55_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -13958,7 +14622,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -13991,21 +14655,21 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB51_1
+; GFX11-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -14032,13 +14696,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB51_1
+; GFX10-NEXT: s_cbranch_execnz .LBB55_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -14047,7 +14711,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -14072,13 +14736,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -14087,7 +14751,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -14112,13 +14776,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB51_1
+; GFX908-NEXT: s_cbranch_execnz .LBB55_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -14127,7 +14791,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -14155,12 +14819,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB51_1
+; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -14176,7 +14840,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -14198,14 +14862,14 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB51_1
+; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -14221,7 +14885,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -14244,7 +14908,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB51_1
+; GFX6-NEXT: s_cbranch_execnz .LBB55_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v3
@@ -14252,12 +14916,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -14268,7 +14932,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -14300,13 +14964,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB52_1
+; GFX12-NEXT: s_cbranch_execnz .LBB56_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -14315,7 +14979,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -14342,13 +15006,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB52_1
+; GFX940-NEXT: s_cbranch_execnz .LBB56_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
@@ -14357,7 +15021,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -14390,21 +15054,21 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB52_1
+; GFX11-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -14431,13 +15095,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB52_1
+; GFX10-NEXT: s_cbranch_execnz .LBB56_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -14446,7 +15110,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -14471,13 +15135,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -14486,7 +15150,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -14511,13 +15175,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB52_1
+; GFX908-NEXT: s_cbranch_execnz .LBB56_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -14526,7 +15190,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -14554,12 +15218,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB52_1
+; GFX8-NEXT: s_cbranch_execnz .LBB56_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -14579,7 +15243,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14601,12 +15265,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB52_1
+; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -14626,7 +15290,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14649,18 +15313,18 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB52_1
+; GFX6-NEXT: s_cbranch_execnz .LBB56_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -14671,7 +15335,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14702,12 +15366,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-NEXT: s_cbranch_execnz .LBB57_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -14716,7 +15380,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14743,12 +15407,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB53_1
+; GFX940-NEXT: s_cbranch_execnz .LBB57_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
@@ -14757,7 +15421,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14789,20 +15453,20 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14829,12 +15493,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB53_1
+; GFX10-NEXT: s_cbranch_execnz .LBB57_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -14843,7 +15507,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14868,12 +15532,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -14882,7 +15546,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14907,19 +15571,19 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB53_1
+; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14947,12 +15611,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB53_1
+; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -14968,7 +15632,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -14990,12 +15654,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB53_1
+; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -15011,7 +15675,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15034,17 +15698,17 @@ define void @global_agent_atomic_fmax_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB53_1
+; GFX6-NEXT: s_cbranch_execnz .LBB57_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -15055,7 +15719,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15086,12 +15750,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-NEXT: s_cbranch_execnz .LBB58_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15100,7 +15764,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15127,12 +15791,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB54_1
+; GFX940-NEXT: s_cbranch_execnz .LBB58_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -15141,7 +15805,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15173,20 +15837,20 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15213,12 +15877,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB54_1
+; GFX10-NEXT: s_cbranch_execnz .LBB58_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15227,7 +15891,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15252,12 +15916,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15266,7 +15930,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15291,12 +15955,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB54_1
+; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -15305,7 +15969,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15333,12 +15997,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB54_1
+; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -15354,7 +16018,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15376,12 +16040,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB54_1
+; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -15397,7 +16061,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15420,18 +16084,18 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB54_1
+; GFX6-NEXT: s_cbranch_execnz .LBB58_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -15442,7 +16106,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15473,12 +16137,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-NEXT: s_cbranch_execnz .LBB59_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -15487,7 +16151,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15514,12 +16178,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB55_1
+; GFX940-NEXT: s_cbranch_execnz .LBB59_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
@@ -15528,7 +16192,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15560,20 +16224,20 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15600,12 +16264,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB55_1
+; GFX10-NEXT: s_cbranch_execnz .LBB59_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -15614,7 +16278,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15639,12 +16303,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -15653,7 +16317,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15678,12 +16342,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB55_1
+; GFX908-NEXT: s_cbranch_execnz .LBB59_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -15692,7 +16356,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15720,12 +16384,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB55_1
+; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -15745,7 +16409,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15767,12 +16431,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB55_1
+; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -15792,7 +16456,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15815,18 +16479,18 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB55_1
+; GFX6-NEXT: s_cbranch_execnz .LBB59_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -15837,7 +16501,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -15869,13 +16533,13 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15884,7 +16548,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -15911,13 +16575,13 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB56_1
+; GFX940-NEXT: s_cbranch_execnz .LBB60_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -15926,7 +16590,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -15959,21 +16623,21 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -16000,13 +16664,13 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB56_1
+; GFX10-NEXT: s_cbranch_execnz .LBB60_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16015,7 +16679,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -16042,13 +16706,13 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16057,7 +16721,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -16082,13 +16746,13 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB56_1
+; GFX908-NEXT: s_cbranch_execnz .LBB60_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -16097,7 +16761,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -16125,12 +16789,12 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB56_1
+; GFX8-NEXT: s_cbranch_execnz .LBB60_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -16146,7 +16810,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -16168,14 +16832,14 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB56_1
+; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -16191,7 +16855,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -16214,7 +16878,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB56_1
+; GFX6-NEXT: s_cbranch_execnz .LBB60_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v3
@@ -16222,12 +16886,12 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
+ %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -16238,7 +16902,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16269,12 +16933,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-NEXT: s_cbranch_execnz .LBB61_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16283,7 +16947,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16310,12 +16974,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB57_1
+; GFX940-NEXT: s_cbranch_execnz .LBB61_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -16324,7 +16988,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16356,20 +17020,20 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16396,12 +17060,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB57_1
+; GFX10-NEXT: s_cbranch_execnz .LBB61_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16410,7 +17074,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16437,12 +17101,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16451,7 +17115,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16476,12 +17140,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB57_1
+; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -16490,7 +17154,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16518,12 +17182,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB57_1
+; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -16539,7 +17203,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -16561,12 +17225,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB57_1
+; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -16582,7 +17246,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -16605,15 +17269,17 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB57_1
+; GFX6-NEXT: s_cbranch_execnz .LBB61_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 915ce7433f5b0..91a8ac7c935b6 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -13,8 +13,8 @@
; float
; --------------------------------------------------------------------
-define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f32:
+define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -27,7 +27,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -52,7 +52,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -62,7 +62,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -72,7 +72,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -96,7 +96,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -120,7 +120,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -144,7 +144,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -157,7 +157,7 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f32:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -170,12 +170,12 @@ define float @global_agent_atomic_fmin_ret_f32(ptr addrspace(1) %ptr, float %val
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -188,7 +188,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -213,7 +213,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -223,7 +223,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -233,7 +233,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -257,7 +257,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -281,7 +281,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -306,7 +306,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -319,7 +319,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -333,12 +333,12 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -351,7 +351,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -376,7 +376,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -386,7 +386,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -396,7 +396,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -420,7 +420,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -444,7 +444,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -469,7 +469,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -482,7 +482,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -496,12 +496,12 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg(ptr addrspace(1) %
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f32:
+define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -514,7 +514,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -538,7 +538,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -548,7 +548,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -558,7 +558,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -581,7 +581,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -604,7 +604,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -627,7 +627,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -639,7 +639,7 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f32:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -651,12 +651,12 @@ define void @global_agent_atomic_fmin_noret_f32(ptr addrspace(1) %ptr, float %va
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -669,7 +669,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -693,7 +693,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -703,7 +703,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -713,7 +713,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -736,7 +736,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -759,7 +759,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -784,7 +784,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -796,7 +796,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -809,12 +809,12 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -827,7 +827,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -851,7 +851,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -861,7 +861,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -871,7 +871,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -894,7 +894,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -917,7 +917,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -942,7 +942,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -954,7 +954,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -967,12 +967,12 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+define float @global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1003,7 +1003,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1028,7 +1028,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -1056,7 +1056,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1082,7 +1082,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1108,7 +1108,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1132,7 +1132,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -1157,7 +1157,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -1187,7 +1187,7 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -1219,12 +1219,12 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+define void @global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1254,7 +1254,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1278,7 +1278,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -1305,7 +1305,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1330,7 +1330,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1355,7 +1355,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -1378,7 +1378,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -1403,7 +1403,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -1432,7 +1432,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -1463,16 +1463,12 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-; --------------------------------------------------------------------
-; float with ftz/daz
-; --------------------------------------------------------------------
-
-define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1485,7 +1481,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -1510,7 +1506,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1520,7 +1516,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1530,7 +1526,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -1554,7 +1550,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -1578,7 +1574,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
@@ -1602,7 +1598,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -1615,7 +1611,7 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -1628,12 +1624,12 @@ define float @global_agent_atomic_fmin_ret_f32__ftz(ptr addrspace(1) %ptr, float
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1641,15 +1637,15 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_min_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1659,7 +1655,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1671,30 +1667,30 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_min_f32 v0, v[0:1], v2, off offset:2044 glc
+; GFX11-NEXT: global_atomic_min_f32 v0, v[0:1], v2, off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off offset:2044 glc
+; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1703,7 +1699,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1715,10 +1711,10 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
@@ -1727,7 +1723,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -1739,64 +1735,66 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret float %result
}
-define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; --------------------------------------------------------------------
+; float with ftz/daz
+; --------------------------------------------------------------------
+
+define float @global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1804,15 +1802,15 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_min_num_f32 v0, v[0:1], v2, off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
@@ -1822,7 +1820,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1834,30 +1832,30 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_min_f32 v0, v[0:1], v2, off offset:-2048 glc
+; GFX11-NEXT: global_atomic_min_f32 v0, v[0:1], v2, off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off offset:-2048 glc
+; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
@@ -1866,7 +1864,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
@@ -1878,10 +1876,10 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
@@ -1890,7 +1888,7 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -1902,64 +1900,62 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz(ptr addrspace
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
}
-define void @global_agent_atomic_fmin_noret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -1967,154 +1963,162 @@ define void @global_agent_atomic_fmin_noret_f32__ftz(ptr addrspace(1) %ptr, floa
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_min_num_f32 v[0:1], v2, off
-; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_min_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB11_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_min_f32 v[0:1], v2, off
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_min_f32 v0, v[0:1], v2, off offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin v[0:1], v2, off
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off offset:2044 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
+ %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2122,157 +2126,162 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspac
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_min_num_f32 v[0:1], v2, off offset:2044
-; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_min_num_f32 v0, v[0:1], v2, off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB12_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_min_f32 v[0:1], v2, off offset:2044
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_min_f32 v0, v[0:1], v2, off offset:-2048 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin v[0:1], v2, off offset:2044
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off offset:-2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
- ret void
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
+ %result = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
}
-define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+define void @global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -2280,15 +2289,15 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_min_num_f32 v[0:1], v2, off offset:-2048
+; GFX12-NEXT: global_atomic_min_num_f32 v[0:1], v2, off
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2297,7 +2306,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2309,30 +2318,30 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_min_f32 v[0:1], v2, off offset:-2048
+; GFX11-NEXT: global_atomic_min_f32 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin v[0:1], v2, off offset:-2048
+; GFX10-NEXT: global_atomic_fmin v[0:1], v2, off
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2340,7 +2349,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2352,10 +2361,10 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
@@ -2363,7 +2372,7 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2375,11 +2384,9 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
@@ -2400,322 +2407,210 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz(ptr addrspac
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define float @global_system_atomic_fmin_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-NEXT: global_atomic_min_num_f32 v[0:1], v2, off offset:2044
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
-; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v5, v3
-; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: buffer_inv sc0 sc1
-; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB14_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_atomic_min_f32 v[0:1], v2, off offset:2044
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
-; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmin v[0:1], v2, off offset:2044
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB14_1
-; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; GFX7-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v3, v4
-; GFX7-NEXT: v_mov_b32_e32 v4, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB14_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; GFX6-NEXT: v_min_f32_e32 v4, v3, v2
-; GFX6-NEXT: v_mov_b32_e32 v3, v4
-; GFX6-NEXT: v_mov_b32_e32 v4, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64 offset:2044
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB14_1
-; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst
- ret float %result
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
-; GFX12-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: global_atomic_min_num_f32 v[0:1], v2, off offset:-2048
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
@@ -2723,10 +2618,10 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX940-NEXT: buffer_wbl2 sc0 sc1
-; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
-; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
@@ -2736,62 +2631,30 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
-; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_atomic_min_f32 v[0:1], v2, off offset:-2048
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
-; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmin v[0:1], v2, off offset:-2048
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB15_1
-; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
@@ -2799,10 +2662,8 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: buffer_wbl2
-; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2813,10 +2674,10 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
@@ -2824,7 +2685,7 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -2836,11 +2697,11 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
@@ -2861,7 +2722,222 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
+; GFX7-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
+; GFX6-NEXT: buffer_atomic_fmin v2, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define float @global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v3, v4, v4
+; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX940-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX940-NEXT: buffer_wbl2 sc0 sc1
+; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB16_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB16_1
+; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_mov_b32_e32 v0, v3
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v3
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[3:4]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; GFX8-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -2870,77 +2946,666 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz(ptr addrspa
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX7-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX7-NEXT: v_mov_b32_e32 v6, v3
-; GFX7-NEXT: v_mov_b32_e32 v5, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX7-NEXT: v_mov_b32_e32 v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_cbranch_execnz .LBB16_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v0, v3
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v5, v3
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v5
+; GFX6-NEXT: v_min_f32_e32 v4, v3, v2
+; GFX6-NEXT: v_mov_b32_e32 v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_cbranch_execnz .LBB16_1
+; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v0, v3
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
+ %result = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret float %result
+}
+
+define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
+; GFX12-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_max_num_f32_e32 v4, v2, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_max_num_f32_e32 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX940-NEXT: s_mov_b64 s[0:1], 0
+; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX940-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX940-NEXT: buffer_wbl2 sc0 sc1
+; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc0 sc1
+; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v3, v2
+; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_cbranch_execnz .LBB17_1
+; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB17_1
+; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
+; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX7-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX7-NEXT: v_mov_b32_e32 v6, v3
+; GFX7-NEXT: v_mov_b32_e32 v5, v2
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v3, v5
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
+; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GFX6-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v6, v3
+; GFX6-NEXT: v_mov_b32_e32 v5, v2
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v3, v5
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_cbranch_execnz .LBB17_1
+; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+; --------------------------------------------------------------------
+; double
+; --------------------------------------------------------------------
+
+define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
+}
+
+define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: buffer_wbl2 sc1
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:2040 sc0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: buffer_inv sc1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off offset:2040 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:2040 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX908: ; %bb.0:
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB19_1
+; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v3, v5
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB15_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
-; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; GFX6-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v6, v3
-; GFX6-NEXT: v_mov_b32_e32 v5, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v5
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB15_1
-; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, float %val seq_cst
- ret void
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
+ %result = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret double %result
}
-; --------------------------------------------------------------------
-; double
-; --------------------------------------------------------------------
-
-define double @global_agent_atomic_fmin_ret_f64(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f64:
+define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
@@ -2948,35 +3613,35 @@ define double @global_agent_atomic_fmin_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: s_cbranch_execnz .LBB20_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
@@ -2984,7 +3649,7 @@ define double @global_agent_atomic_fmin_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -2992,89 +3657,89 @@ define double @global_agent_atomic_fmin_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: s_cbranch_execnz .LBB20_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off glc
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_1
+; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB16_1
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
@@ -3082,13 +3747,13 @@ define double @global_agent_atomic_fmin_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f64:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
@@ -3096,373 +3761,355 @@ define double @global_agent_atomic_fmin_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
+ %result = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret double %result
}
-define double @global_agent_atomic_fmin_ret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:2040 sc0
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off offset:2040 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[2:3], off
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:2040 glc
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB17_1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_mov_b32_e32 v0, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_mov_b32_e32 v0, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
- %result = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define double @global_agent_atomic_fmin_ret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-NEXT: s_cbranch_execnz .LBB22_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 sc0
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:2040
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-NEXT: s_cbranch_execnz .LBB22_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[2:3], off offset:2040
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:2040
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_1
+; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v0, v4
-; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_mov_b32_e32 v0, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_mov_b32_e32 v0, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
- %result = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret double %result
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
}
-define void @global_agent_atomic_fmin_noret_f64(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f64:
+define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
@@ -3470,34 +4117,34 @@ define void @global_agent_atomic_fmin_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB19_1
+; GFX12-NEXT: s_cbranch_execnz .LBB23_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:-2048
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -3506,41 +4153,41 @@ define void @global_agent_atomic_fmin_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-NEXT: s_cbranch_execnz .LBB23_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[2:3], off
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[2:3], off offset:-2048
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:-2048
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
@@ -3548,18 +4195,20 @@ define void @global_agent_atomic_fmin_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB19_1
+; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
@@ -3572,373 +4221,388 @@ define void @global_agent_atomic_fmin_noret_f64(ptr addrspace(1) %ptr, double %v
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f64:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
+ %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB20_1
+; GFX12-NEXT: s_cbranch_execnz .LBB24_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:2040
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB20_1
+; GFX11-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[2:3], off offset:2040
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:2040
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB20_1
+; GFX908-NEXT: s_cbranch_execnz .LBB24_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB20_1
+; GFX8-NEXT: s_cbranch_execnz .LBB24_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
- %unused = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+ ret double %result
}
-define void @global_agent_atomic_fmin_noret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB21_1
+; GFX12-NEXT: s_cbranch_execnz .LBB25_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:-2048
+; GFX940-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off sc0
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[2:3], off offset:-2048
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[2:3], off offset:-2048
+; GFX90A-NEXT: global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_1
+; GFX908-NEXT: s_cbranch_execnz .LBB25_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_1
+; GFX8-NEXT: s_cbranch_execnz .LBB25_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_movk_i32 s4, 0xf800
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s5, -1
-; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_movk_i32 s4, 0xf800
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s5, -1
-; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s6
+; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: buffer_atomic_fmin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
- %unused = atomicrmw fmin ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
- ret void
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret double %result
}
; --------------------------------------------------------------------
; half
; --------------------------------------------------------------------
-define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f16:
+define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -3956,7 +4620,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -3977,13 +4641,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB22_1
+; GFX12-NEXT: s_cbranch_execnz .LBB26_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -3996,7 +4660,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4012,13 +4676,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB22_1
+; GFX940-NEXT: s_cbranch_execnz .LBB26_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -4032,7 +4696,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4054,13 +4718,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -4072,7 +4736,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4089,13 +4753,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB22_1
+; GFX10-NEXT: s_cbranch_execnz .LBB26_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -4108,7 +4772,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4123,13 +4787,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -4142,7 +4806,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4157,13 +4821,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB22_1
+; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -4176,7 +4840,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4192,13 +4856,13 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB22_1
+; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -4215,7 +4879,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
; GFX7-NEXT: v_not_b32_e32 v7, v2
-; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -4234,14 +4898,14 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB22_1
+; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f16:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -4258,7 +4922,7 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
; GFX6-NEXT: v_not_b32_e32 v7, v2
-; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -4277,19 +4941,19 @@ define half @global_agent_atomic_fmin_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB22_1
+; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4308,7 +4972,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -4329,13 +4993,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB23_1
+; GFX12-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -4350,7 +5014,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4366,13 +5030,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB23_1
+; GFX940-NEXT: s_cbranch_execnz .LBB27_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4387,7 +5051,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4409,13 +5073,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -4428,7 +5092,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4445,13 +5109,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB23_1
+; GFX10-NEXT: s_cbranch_execnz .LBB27_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4465,7 +5129,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4480,13 +5144,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -4500,7 +5164,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4515,13 +5179,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB23_1
+; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -4535,7 +5199,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4551,13 +5215,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB23_1
+; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -4575,7 +5239,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX7-NEXT: v_not_b32_e32 v8, v2
-; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -4594,14 +5258,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB23_1
+; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -4619,7 +5283,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX6-NEXT: v_not_b32_e32 v8, v2
-; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -4639,7 +5303,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB23_1
+; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -4647,12 +5311,12 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -4671,7 +5335,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -4692,13 +5356,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB24_1
+; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -4714,7 +5378,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -4730,13 +5394,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB24_1
+; GFX940-NEXT: s_cbranch_execnz .LBB28_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4751,7 +5415,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -4773,13 +5437,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -4792,7 +5456,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -4809,13 +5473,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB24_1
+; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4829,7 +5493,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -4844,13 +5508,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4864,7 +5528,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -4879,13 +5543,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_1
+; GFX908-NEXT: s_cbranch_execnz .LBB28_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -4899,7 +5563,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -4915,13 +5579,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB24_1
+; GFX8-NEXT: s_cbranch_execnz .LBB28_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -4939,7 +5603,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX7-NEXT: v_not_b32_e32 v8, v2
-; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -4958,14 +5622,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB24_1
+; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -4983,7 +5647,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX6-NEXT: v_not_b32_e32 v8, v2
-; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -5003,7 +5667,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB24_1
+; GFX6-NEXT: s_cbranch_execnz .LBB28_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -5011,12 +5675,12 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
- %result = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f16:
+define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5034,7 +5698,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5055,12 +5719,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB25_1
+; GFX12-NEXT: s_cbranch_execnz .LBB29_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -5073,7 +5737,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX940-NEXT: v_not_b32_e32 v6, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -5089,12 +5753,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB25_1
+; GFX940-NEXT: s_cbranch_execnz .LBB29_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v0
@@ -5108,7 +5772,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5130,12 +5794,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-NEXT: s_cbranch_execnz .LBB29_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -5147,7 +5811,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5164,12 +5828,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB25_1
+; GFX10-NEXT: s_cbranch_execnz .LBB29_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -5182,7 +5846,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
@@ -5197,12 +5861,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -5215,7 +5879,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5230,12 +5894,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_1
+; GFX908-NEXT: s_cbranch_execnz .LBB29_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -5248,7 +5912,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -5264,12 +5928,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB25_1
+; GFX8-NEXT: s_cbranch_execnz .LBB29_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -5286,7 +5950,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
@@ -5305,12 +5969,12 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB25_1
+; GFX7-NEXT: s_cbranch_execnz .LBB29_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f16:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -5327,7 +5991,7 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
@@ -5347,17 +6011,17 @@ define void @global_agent_atomic_fmin_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB25_1
+; GFX6-NEXT: s_cbranch_execnz .LBB29_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5376,7 +6040,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5397,12 +6061,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB26_1
+; GFX12-NEXT: s_cbranch_execnz .LBB30_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -5417,7 +6081,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5433,12 +6097,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB26_1
+; GFX940-NEXT: s_cbranch_execnz .LBB30_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -5453,7 +6117,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5475,12 +6139,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB26_1
+; GFX11-NEXT: s_cbranch_execnz .LBB30_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -5493,7 +6157,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5510,12 +6174,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB26_1
+; GFX10-NEXT: s_cbranch_execnz .LBB30_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5529,7 +6193,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5544,12 +6208,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -5563,7 +6227,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5578,12 +6242,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB26_1
+; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -5597,7 +6261,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5613,12 +6277,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB26_1
+; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -5636,7 +6300,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5655,12 +6319,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB26_1
+; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -5678,7 +6342,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX6-NEXT: v_not_b32_e32 v6, v2
-; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5698,18 +6362,18 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB26_1
+; GFX6-NEXT: s_cbranch_execnz .LBB30_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -5728,7 +6392,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5749,12 +6413,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB27_1
+; GFX12-NEXT: s_cbranch_execnz .LBB31_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -5770,7 +6434,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5786,12 +6450,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB27_1
+; GFX940-NEXT: s_cbranch_execnz .LBB31_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5806,7 +6470,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5828,12 +6492,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB27_1
+; GFX11-NEXT: s_cbranch_execnz .LBB31_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -5846,7 +6510,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5863,12 +6527,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB27_1
+; GFX10-NEXT: s_cbranch_execnz .LBB31_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5882,7 +6546,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5897,12 +6561,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5916,7 +6580,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5931,12 +6595,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB27_1
+; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -5950,7 +6614,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -5966,12 +6630,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB27_1
+; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -5989,7 +6653,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6008,12 +6672,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB27_1
+; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -6031,7 +6695,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX6-NEXT: v_not_b32_e32 v6, v2
-; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -6051,18 +6715,18 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB27_1
+; GFX6-NEXT: s_cbranch_execnz .LBB31_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
- %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6072,7 +6736,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -6090,20 +6754,20 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB28_1
+; GFX12-NEXT: s_cbranch_execnz .LBB32_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -6117,19 +6781,19 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB28_1
+; GFX940-NEXT: s_cbranch_execnz .LBB32_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -6148,19 +6812,19 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB28_1
+; GFX11-NEXT: s_cbranch_execnz .LBB32_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -6176,20 +6840,20 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB28_1
+; GFX10-NEXT: s_cbranch_execnz .LBB32_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -6202,20 +6866,20 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -6228,13 +6892,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB28_1
+; GFX908-NEXT: s_cbranch_execnz .LBB32_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -6242,7 +6906,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -6256,12 +6920,12 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB28_1
+; GFX8-NEXT: s_cbranch_execnz .LBB32_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -6272,7 +6936,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
-; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6289,13 +6953,13 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB28_1
+; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -6306,7 +6970,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
-; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6324,19 +6988,19 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB28_1
+; GFX6-NEXT: s_cbranch_execnz .LBB32_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6346,7 +7010,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f16_e32 v2, v3, v3
@@ -6364,19 +7028,19 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB29_1
+; GFX12-NEXT: s_cbranch_execnz .LBB33_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v4, v2, v2
; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
-; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6390,18 +7054,18 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB29_1
+; GFX940-NEXT: s_cbranch_execnz .LBB33_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6420,18 +7084,18 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB29_1
+; GFX11-NEXT: s_cbranch_execnz .LBB33_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6447,19 +7111,19 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB29_1
+; GFX10-NEXT: s_cbranch_execnz .LBB33_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6472,19 +7136,19 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
-; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6497,12 +7161,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB29_1
+; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -6510,7 +7174,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
-; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
@@ -6524,12 +7188,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB29_1
+; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -6540,7 +7204,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6557,12 +7221,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB29_1
+; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -6573,7 +7237,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
@@ -6591,18 +7255,18 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos(ptr addrs
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB29_1
+; GFX6-NEXT: s_cbranch_execnz .LBB33_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6621,7 +7285,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -6642,13 +7306,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB30_1
+; GFX12-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -6663,7 +7327,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX940-NEXT: v_not_b32_e32 v4, v4
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -6679,13 +7343,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB30_1
+; GFX940-NEXT: s_cbranch_execnz .LBB34_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6700,7 +7364,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -6722,13 +7386,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB30_1
+; GFX11-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -6741,7 +7405,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -6758,13 +7422,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB30_1
+; GFX10-NEXT: s_cbranch_execnz .LBB34_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6778,7 +7442,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX90A-NEXT: v_not_b32_e32 v4, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -6795,13 +7459,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -6815,7 +7479,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -6830,13 +7494,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_1
+; GFX908-NEXT: s_cbranch_execnz .LBB34_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -6850,7 +7514,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -6866,13 +7530,13 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB30_1
+; GFX8-NEXT: s_cbranch_execnz .LBB34_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -6890,7 +7554,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX7-NEXT: v_not_b32_e32 v8, v2
-; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -6909,14 +7573,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB30_1
+; GFX7-NEXT: s_cbranch_execnz .LBB34_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -6934,7 +7598,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
; GFX6-NEXT: v_not_b32_e32 v8, v2
-; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -6954,7 +7618,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB30_1
+; GFX6-NEXT: s_cbranch_execnz .LBB34_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -6962,12 +7626,12 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmin ptr addrspace(1) %gep, half %val seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret half %result
}
-define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -6986,7 +7650,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7007,12 +7671,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB31_1
+; GFX12-NEXT: s_cbranch_execnz .LBB35_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -7027,7 +7691,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_not_b32_e32 v5, v5
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7043,12 +7707,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB31_1
+; GFX940-NEXT: s_cbranch_execnz .LBB35_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -7063,7 +7727,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7085,12 +7749,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB31_1
+; GFX11-NEXT: s_cbranch_execnz .LBB35_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -7103,7 +7767,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7120,12 +7784,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB31_1
+; GFX10-NEXT: s_cbranch_execnz .LBB35_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -7139,7 +7803,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7156,12 +7820,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -7175,7 +7839,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7190,12 +7854,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB31_1
+; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -7209,7 +7873,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7225,12 +7889,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB31_1
+; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -7248,7 +7912,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX7-NEXT: v_not_b32_e32 v6, v2
-; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7267,12 +7931,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB31_1
+; GFX7-NEXT: s_cbranch_execnz .LBB35_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -7290,7 +7954,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
; GFX6-NEXT: v_not_b32_e32 v6, v2
-; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -7310,13 +7974,13 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB31_1
+; GFX6-NEXT: s_cbranch_execnz .LBB35_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -7324,8 +7988,8 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos(ptr addrspace(1)
; bfloat
; --------------------------------------------------------------------
-define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16:
+define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7342,7 +8006,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -7370,13 +8034,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -7390,7 +8054,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -7412,13 +8076,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB32_1
+; GFX940-NEXT: s_cbranch_execnz .LBB36_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -7432,7 +8096,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7461,13 +8125,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -7479,7 +8143,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7500,13 +8164,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB32_1
+; GFX10-NEXT: s_cbranch_execnz .LBB36_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -7520,7 +8184,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7539,13 +8203,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -7559,7 +8223,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7578,13 +8242,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB32_1
+; GFX908-NEXT: s_cbranch_execnz .LBB36_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -7597,7 +8261,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -7618,13 +8282,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB32_1
+; GFX8-NEXT: s_cbranch_execnz .LBB36_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -7641,7 +8305,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -7661,14 +8325,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB32_1
+; GFX7-NEXT: s_cbranch_execnz .LBB36_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -7685,7 +8349,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
@@ -7705,19 +8369,19 @@ define bfloat @global_agent_atomic_fmin_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB32_1
+; GFX6-NEXT: s_cbranch_execnz .LBB36_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -7736,7 +8400,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -7764,13 +8428,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -7786,7 +8450,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -7808,13 +8472,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB33_1
+; GFX940-NEXT: s_cbranch_execnz .LBB37_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7830,7 +8494,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -7859,13 +8523,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -7878,7 +8542,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -7899,13 +8563,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB33_1
+; GFX10-NEXT: s_cbranch_execnz .LBB37_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7920,7 +8584,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -7939,13 +8603,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -7960,7 +8624,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -7979,13 +8643,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB33_1
+; GFX908-NEXT: s_cbranch_execnz .LBB37_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -7999,7 +8663,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -8020,13 +8684,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB33_1
+; GFX8-NEXT: s_cbranch_execnz .LBB37_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -8044,7 +8708,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8064,14 +8728,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB33_1
+; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -8089,7 +8753,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8110,7 +8774,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB33_1
+; GFX6-NEXT: s_cbranch_execnz .LBB37_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -8118,12 +8782,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8142,7 +8806,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -8170,13 +8834,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -8193,7 +8857,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -8215,13 +8879,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB34_1
+; GFX940-NEXT: s_cbranch_execnz .LBB38_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -8237,7 +8901,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -8266,13 +8930,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
@@ -8285,7 +8949,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -8306,13 +8970,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB34_1
+; GFX10-NEXT: s_cbranch_execnz .LBB38_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -8327,7 +8991,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -8346,13 +9010,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
@@ -8367,7 +9031,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -8386,13 +9050,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB34_1
+; GFX908-NEXT: s_cbranch_execnz .LBB38_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -8406,7 +9070,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -8427,13 +9091,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB34_1
+; GFX8-NEXT: s_cbranch_execnz .LBB38_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -8451,7 +9115,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8471,14 +9135,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB34_1
+; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -8496,7 +9160,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -8517,7 +9181,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB34_1
+; GFX6-NEXT: s_cbranch_execnz .LBB38_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -8525,12 +9189,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
- %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16:
+define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8547,7 +9211,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8574,12 +9238,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v3, v0
@@ -8593,7 +9257,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8615,12 +9279,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v5, v4
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB35_1
+; GFX940-NEXT: s_cbranch_execnz .LBB39_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
@@ -8634,7 +9298,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v6, v3
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8662,12 +9326,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
@@ -8679,7 +9343,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8700,12 +9364,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB35_1
+; GFX10-NEXT: s_cbranch_execnz .LBB39_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
@@ -8719,7 +9383,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8738,12 +9402,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, v0
@@ -8757,7 +9421,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8776,12 +9440,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB35_1
+; GFX908-NEXT: s_cbranch_execnz .LBB39_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
@@ -8794,7 +9458,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8815,12 +9479,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB35_1
+; GFX8-NEXT: s_cbranch_execnz .LBB39_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v0
@@ -8837,7 +9501,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8857,12 +9521,12 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v4, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB35_1
+; GFX7-NEXT: s_cbranch_execnz .LBB39_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v3, v0
@@ -8879,7 +9543,7 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
@@ -8900,17 +9564,17 @@ define void @global_agent_atomic_fmin_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v4, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB35_1
+; GFX6-NEXT: s_cbranch_execnz .LBB39_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -8929,7 +9593,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -8956,12 +9620,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: s_cbranch_execnz .LBB40_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -8977,7 +9641,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -8999,12 +9663,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB36_1
+; GFX940-NEXT: s_cbranch_execnz .LBB40_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9020,7 +9684,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9048,12 +9712,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: s_cbranch_execnz .LBB40_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -9066,7 +9730,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9087,12 +9751,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB36_1
+; GFX10-NEXT: s_cbranch_execnz .LBB40_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9107,7 +9771,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9126,12 +9790,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -9146,7 +9810,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9165,12 +9829,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB36_1
+; GFX908-NEXT: s_cbranch_execnz .LBB40_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -9184,7 +9848,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9205,12 +9869,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB36_1
+; GFX8-NEXT: s_cbranch_execnz .LBB40_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -9228,7 +9892,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9248,12 +9912,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB36_1
+; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -9271,7 +9935,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9292,18 +9956,18 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB36_1
+; GFX6-NEXT: s_cbranch_execnz .LBB40_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9322,7 +9986,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9349,12 +10013,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: s_cbranch_execnz .LBB41_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_movk_i32 s0, 0xf800
@@ -9371,7 +10035,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9393,12 +10057,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB37_1
+; GFX940-NEXT: s_cbranch_execnz .LBB41_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -9414,7 +10078,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9442,12 +10106,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: s_cbranch_execnz .LBB41_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
@@ -9460,7 +10124,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9481,12 +10145,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB37_1
+; GFX10-NEXT: s_cbranch_execnz .LBB41_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -9501,7 +10165,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9520,12 +10184,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
@@ -9540,7 +10204,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9559,12 +10223,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB37_1
+; GFX908-NEXT: s_cbranch_execnz .LBB41_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -9578,7 +10242,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -9599,12 +10263,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB37_1
+; GFX8-NEXT: s_cbranch_execnz .LBB41_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -9622,7 +10286,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9642,12 +10306,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB37_1
+; GFX7-NEXT: s_cbranch_execnz .LBB41_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
@@ -9665,7 +10329,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -9686,18 +10350,18 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB37_1
+; GFX6-NEXT: s_cbranch_execnz .LBB41_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
- %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -9707,7 +10371,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -9732,13 +10396,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -9746,7 +10410,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -9767,20 +10431,20 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB38_1
+; GFX940-NEXT: s_cbranch_execnz .LBB42_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -9806,19 +10470,19 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -9839,13 +10503,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB38_1
+; GFX10-NEXT: s_cbranch_execnz .LBB42_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -9853,7 +10517,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -9872,13 +10536,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -9886,7 +10550,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -9905,13 +10569,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB38_1
+; GFX908-NEXT: s_cbranch_execnz .LBB42_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -9919,7 +10583,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -9939,12 +10603,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB38_1
+; GFX8-NEXT: s_cbranch_execnz .LBB42_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -9955,7 +10619,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -9973,13 +10637,13 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB38_1
+; GFX7-NEXT: s_cbranch_execnz .LBB42_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -9990,7 +10654,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10009,19 +10673,19 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB38_1
+; GFX6-NEXT: s_cbranch_execnz .LBB42_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10031,7 +10695,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10055,12 +10719,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: s_cbranch_execnz .LBB43_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -10068,7 +10732,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
-; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10089,19 +10753,19 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB39_1
+; GFX940-NEXT: s_cbranch_execnz .LBB43_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10126,18 +10790,18 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: s_cbranch_execnz .LBB43_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10158,12 +10822,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB39_1
+; GFX10-NEXT: s_cbranch_execnz .LBB43_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -10171,7 +10835,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10190,12 +10854,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
@@ -10203,7 +10867,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
-; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10222,12 +10886,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB39_1
+; GFX908-NEXT: s_cbranch_execnz .LBB43_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
@@ -10235,7 +10899,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10255,12 +10919,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB39_1
+; GFX8-NEXT: s_cbranch_execnz .LBB43_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -10271,7 +10935,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10289,12 +10953,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB39_1
+; GFX7-NEXT: s_cbranch_execnz .LBB43_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -10305,7 +10969,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -10324,18 +10988,18 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB39_1
+; GFX6-NEXT: s_cbranch_execnz .LBB43_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
ret void
}
-define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10354,7 +11018,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v5
@@ -10382,13 +11046,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -10404,7 +11068,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v5
@@ -10426,13 +11090,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB40_1
+; GFX940-NEXT: s_cbranch_execnz .LBB44_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -10448,7 +11112,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v4, v4
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v5
@@ -10477,13 +11141,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
@@ -10496,7 +11160,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
; GFX10-NEXT: v_not_b32_e32 v4, v4
-; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
@@ -10517,13 +11181,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB40_1
+; GFX10-NEXT: s_cbranch_execnz .LBB44_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -10538,7 +11202,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v5
@@ -10559,13 +11223,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
@@ -10580,7 +11244,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
@@ -10599,13 +11263,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB40_1
+; GFX908-NEXT: s_cbranch_execnz .LBB44_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
@@ -10619,7 +11283,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
@@ -10640,13 +11304,13 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB40_1
+; GFX8-NEXT: s_cbranch_execnz .LBB44_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -10664,7 +11328,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -10684,14 +11348,14 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB40_1
+; GFX7-NEXT: s_cbranch_execnz .LBB44_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -10709,7 +11373,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
@@ -10730,7 +11394,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB40_1
+; GFX6-NEXT: s_cbranch_execnz .LBB44_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
@@ -10738,12 +11402,12 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
}
-define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -10762,7 +11426,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_not_b32_e32 v5, v5
-; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -10789,12 +11453,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB41_1
+; GFX12-NEXT: s_cbranch_execnz .LBB45_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
@@ -10810,7 +11474,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX940-NEXT: s_movk_i32 s2, 0x7fff
-; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10832,12 +11496,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB41_1
+; GFX940-NEXT: s_cbranch_execnz .LBB45_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -10853,7 +11517,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v5, v5
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -10881,12 +11545,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB41_1
+; GFX11-NEXT: s_cbranch_execnz .LBB45_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
@@ -10899,7 +11563,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v5, v5
-; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10920,12 +11584,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB41_1
+; GFX10-NEXT: s_cbranch_execnz .LBB45_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -10940,7 +11604,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
-; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10961,12 +11625,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
@@ -10981,7 +11645,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX908-NEXT: s_movk_i32 s6, 0x7fff
-; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -11000,12 +11664,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB41_1
+; GFX908-NEXT: s_cbranch_execnz .LBB45_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
@@ -11019,7 +11683,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -11040,12 +11704,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB41_1
+; GFX8-NEXT: s_cbranch_execnz .LBB45_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -11063,7 +11727,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -11083,12 +11747,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v3, v7
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB41_1
+; GFX7-NEXT: s_cbranch_execnz .LBB45_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
@@ -11106,7 +11770,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
@@ -11127,13 +11791,13 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v3, v7
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB41_1
+; GFX6-NEXT: s_cbranch_execnz .LBB45_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -11141,8 +11805,8 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos(ptr addrspace(1
; <2 x half>
; --------------------------------------------------------------------
-define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_v2f16:
+define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11152,7 +11816,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11167,19 +11831,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB42_1
+; GFX12-NEXT: s_cbranch_execnz .LBB46_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11193,19 +11857,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB42_1
+; GFX940-NEXT: s_cbranch_execnz .LBB46_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11221,19 +11885,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB42_1
+; GFX11-NEXT: s_cbranch_execnz .LBB46_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -11247,19 +11911,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB42_1
+; GFX10-NEXT: s_cbranch_execnz .LBB46_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11271,19 +11935,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11295,20 +11959,20 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB42_1
+; GFX908-NEXT: s_cbranch_execnz .LBB46_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -11323,13 +11987,13 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB42_1
+; GFX8-NEXT: s_cbranch_execnz .LBB46_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -11346,7 +12010,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11371,14 +12035,14 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB42_1
+; GFX7-NEXT: s_cbranch_execnz .LBB46_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_v2f16:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -11395,7 +12059,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11421,19 +12085,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16(ptr addrspace(1) %ptr, <2
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB42_1
+; GFX6-NEXT: s_cbranch_execnz .LBB46_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11443,7 +12107,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11458,19 +12122,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB43_1
+; GFX12-NEXT: s_cbranch_execnz .LBB47_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11484,19 +12148,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB43_1
+; GFX940-NEXT: s_cbranch_execnz .LBB47_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11512,19 +12176,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB43_1
+; GFX11-NEXT: s_cbranch_execnz .LBB47_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -11538,19 +12202,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB43_1
+; GFX10-NEXT: s_cbranch_execnz .LBB47_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11562,19 +12226,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11586,13 +12250,13 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB43_1
+; GFX908-NEXT: s_cbranch_execnz .LBB47_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -11601,7 +12265,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -11616,12 +12280,12 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB43_1
+; GFX8-NEXT: s_cbranch_execnz .LBB47_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -11638,7 +12302,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11663,14 +12327,14 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB43_1
+; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -11687,7 +12351,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -11713,7 +12377,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB43_1
+; GFX6-NEXT: s_cbranch_execnz .LBB47_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
@@ -11721,12 +12385,12 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspa
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -11736,7 +12400,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -11751,19 +12415,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: s_cbranch_execnz .LBB48_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -11777,19 +12441,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB44_1
+; GFX940-NEXT: s_cbranch_execnz .LBB48_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -11805,19 +12469,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: s_cbranch_execnz .LBB48_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -11831,19 +12495,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB44_1
+; GFX10-NEXT: s_cbranch_execnz .LBB48_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -11855,19 +12519,19 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -11879,13 +12543,13 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB44_1
+; GFX908-NEXT: s_cbranch_execnz .LBB48_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -11894,7 +12558,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -11909,12 +12573,12 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB44_1
+; GFX8-NEXT: s_cbranch_execnz .LBB48_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -11935,7 +12599,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v6
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -11960,12 +12624,12 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB44_1
+; GFX7-NEXT: s_cbranch_execnz .LBB48_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -11986,7 +12650,7 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -12012,18 +12676,18 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg(ptr addrspa
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB44_1
+; GFX6-NEXT: s_cbranch_execnz .LBB48_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_v2f16:
+define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12033,7 +12697,7 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -12048,18 +12712,18 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB45_1
+; GFX12-NEXT: s_cbranch_execnz .LBB49_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12073,18 +12737,18 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB45_1
+; GFX940-NEXT: s_cbranch_execnz .LBB49_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12100,18 +12764,18 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB45_1
+; GFX11-NEXT: s_cbranch_execnz .LBB49_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12125,18 +12789,18 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB45_1
+; GFX10-NEXT: s_cbranch_execnz .LBB49_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12148,18 +12812,18 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12171,19 +12835,19 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB45_1
+; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -12198,12 +12862,12 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB45_1
+; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -12220,7 +12884,7 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12245,12 +12909,12 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB45_1
+; GFX7-NEXT: s_cbranch_execnz .LBB49_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_v2f16:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -12267,7 +12931,7 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12293,17 +12957,17 @@ define void @global_agent_atomic_fmin_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB45_1
+; GFX6-NEXT: s_cbranch_execnz .LBB49_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12313,7 +12977,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -12328,18 +12992,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-NEXT: s_cbranch_execnz .LBB50_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12353,18 +13017,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB46_1
+; GFX940-NEXT: s_cbranch_execnz .LBB50_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12380,18 +13044,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-NEXT: s_cbranch_execnz .LBB50_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12405,18 +13069,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB46_1
+; GFX10-NEXT: s_cbranch_execnz .LBB50_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12428,18 +13092,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12451,12 +13115,12 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB46_1
+; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -12465,7 +13129,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -12480,12 +13144,12 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB46_1
+; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -12502,7 +13166,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12527,12 +13191,12 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB46_1
+; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -12549,7 +13213,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12575,18 +13239,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB46_1
+; GFX6-NEXT: s_cbranch_execnz .LBB50_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12596,7 +13260,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -12611,18 +13275,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12636,18 +13300,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB47_1
+; GFX940-NEXT: s_cbranch_execnz .LBB51_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12663,18 +13327,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12688,18 +13352,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB47_1
+; GFX10-NEXT: s_cbranch_execnz .LBB51_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12711,18 +13375,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -12734,12 +13398,12 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB47_1
+; GFX908-NEXT: s_cbranch_execnz .LBB51_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -12748,7 +13412,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -12763,12 +13427,12 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB47_1
+; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -12789,7 +13453,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12814,12 +13478,12 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB47_1
+; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -12840,7 +13504,7 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -12866,18 +13530,18 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg(ptr addrspace(1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB47_1
+; GFX6-NEXT: s_cbranch_execnz .LBB51_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -12887,7 +13551,7 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
@@ -12902,19 +13566,19 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-NEXT: s_cbranch_execnz .LBB52_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v5, v3
@@ -12928,19 +13592,19 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB48_1
+; GFX940-NEXT: s_cbranch_execnz .LBB52_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
@@ -12956,19 +13620,19 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-NEXT: s_cbranch_execnz .LBB52_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
@@ -12982,19 +13646,19 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB48_1
+; GFX10-NEXT: s_cbranch_execnz .LBB52_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
@@ -13008,19 +13672,19 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
@@ -13032,13 +13696,13 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB48_1
+; GFX908-NEXT: s_cbranch_execnz .LBB52_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -13047,7 +13711,7 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
-; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -13062,12 +13726,12 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB48_1
+; GFX8-NEXT: s_cbranch_execnz .LBB52_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -13084,7 +13748,7 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -13109,14 +13773,14 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB48_1
+; GFX7-NEXT: s_cbranch_execnz .LBB52_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -13133,7 +13797,7 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
-; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -13159,7 +13823,7 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB48_1
+; GFX6-NEXT: s_cbranch_execnz .LBB52_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
@@ -13167,12 +13831,12 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos(ptr addrsp
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
-define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13182,7 +13846,7 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
@@ -13197,18 +13861,18 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB49_1
+; GFX12-NEXT: s_cbranch_execnz .LBB53_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13222,18 +13886,18 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX940-NEXT: s_cbranch_execnz .LBB49_1
+; GFX940-NEXT: s_cbranch_execnz .LBB53_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13249,18 +13913,18 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB49_1
+; GFX11-NEXT: s_cbranch_execnz .LBB53_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13274,18 +13938,18 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_cbranch_execnz .LBB49_1
+; GFX10-NEXT: s_cbranch_execnz .LBB53_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13299,18 +13963,18 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
@@ -13322,12 +13986,12 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB49_1
+; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -13336,7 +14000,7 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
@@ -13351,12 +14015,12 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB49_1
+; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -13373,7 +14037,7 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -13398,12 +14062,12 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB49_1
+; GFX7-NEXT: s_cbranch_execnz .LBB53_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -13420,7 +14084,7 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
-; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -13446,13 +14110,13 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB49_1
+; GFX6-NEXT: s_cbranch_execnz .LBB53_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -13460,8 +14124,8 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos(ptr addrspace(
; <2 x bfloat>
; --------------------------------------------------------------------
-define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13472,7 +14136,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -13504,13 +14168,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB50_1
+; GFX12-NEXT: s_cbranch_execnz .LBB54_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -13519,7 +14183,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -13546,13 +14210,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB50_1
+; GFX940-NEXT: s_cbranch_execnz .LBB54_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
@@ -13561,7 +14225,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -13594,21 +14258,21 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB50_1
+; GFX11-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -13635,13 +14299,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB50_1
+; GFX10-NEXT: s_cbranch_execnz .LBB54_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -13650,7 +14314,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -13675,13 +14339,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -13690,7 +14354,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -13715,20 +14379,20 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB50_1
+; GFX908-NEXT: s_cbranch_execnz .LBB54_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v3
@@ -13756,13 +14420,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB50_1
+; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -13778,7 +14442,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -13800,14 +14464,14 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB50_1
+; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -13823,7 +14487,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -13846,19 +14510,19 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB50_1
+; GFX6-NEXT: s_cbranch_execnz .LBB54_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v3
; GFX6-NEXT: v_mov_b32_e32 v1, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %result = atomicrmw fmin ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -13869,7 +14533,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -13901,13 +14565,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB51_1
+; GFX12-NEXT: s_cbranch_execnz .LBB55_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -13916,7 +14580,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -13943,13 +14607,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB51_1
+; GFX940-NEXT: s_cbranch_execnz .LBB55_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -13958,7 +14622,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -13991,21 +14655,21 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB51_1
+; GFX11-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -14032,13 +14696,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB51_1
+; GFX10-NEXT: s_cbranch_execnz .LBB55_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -14047,7 +14711,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -14072,13 +14736,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -14087,7 +14751,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -14112,13 +14776,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB51_1
+; GFX908-NEXT: s_cbranch_execnz .LBB55_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -14127,7 +14791,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -14155,12 +14819,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB51_1
+; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -14176,7 +14840,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -14198,14 +14862,14 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB51_1
+; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -14221,7 +14885,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -14244,7 +14908,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB51_1
+; GFX6-NEXT: s_cbranch_execnz .LBB55_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v3
@@ -14252,12 +14916,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -14268,7 +14932,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -14300,13 +14964,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB52_1
+; GFX12-NEXT: s_cbranch_execnz .LBB56_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -14315,7 +14979,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -14342,13 +15006,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB52_1
+; GFX940-NEXT: s_cbranch_execnz .LBB56_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
@@ -14357,7 +15021,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -14390,21 +15054,21 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB52_1
+; GFX11-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -14431,13 +15095,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB52_1
+; GFX10-NEXT: s_cbranch_execnz .LBB56_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -14446,7 +15110,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -14471,13 +15135,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -14486,7 +15150,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -14511,13 +15175,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB52_1
+; GFX908-NEXT: s_cbranch_execnz .LBB56_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
@@ -14526,7 +15190,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -14554,12 +15218,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB52_1
+; GFX8-NEXT: s_cbranch_execnz .LBB56_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -14579,7 +15243,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14601,12 +15265,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB52_1
+; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -14626,7 +15290,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14649,18 +15313,18 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg(ptr addr
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB52_1
+; GFX6-NEXT: s_cbranch_execnz .LBB56_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
- %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -14671,7 +15335,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14702,12 +15366,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-NEXT: s_cbranch_execnz .LBB57_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off
@@ -14716,7 +15380,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14743,12 +15407,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB53_1
+; GFX940-NEXT: s_cbranch_execnz .LBB57_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
@@ -14757,7 +15421,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14789,20 +15453,20 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14829,12 +15493,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB53_1
+; GFX10-NEXT: s_cbranch_execnz .LBB57_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
@@ -14843,7 +15507,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14868,12 +15532,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
@@ -14882,7 +15546,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14907,19 +15571,19 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB53_1
+; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -14947,12 +15611,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB53_1
+; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -14968,7 +15632,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -14990,12 +15654,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB53_1
+; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_v2bf16:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -15011,7 +15675,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15034,17 +15698,17 @@ define void @global_agent_atomic_fmin_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB53_1
+; GFX6-NEXT: s_cbranch_execnz .LBB57_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
- %unused = atomicrmw fmin ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -15055,7 +15719,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15086,12 +15750,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-NEXT: s_cbranch_execnz .LBB58_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15100,7 +15764,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15127,12 +15791,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB54_1
+; GFX940-NEXT: s_cbranch_execnz .LBB58_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -15141,7 +15805,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15173,20 +15837,20 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15213,12 +15877,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB54_1
+; GFX10-NEXT: s_cbranch_execnz .LBB58_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15227,7 +15891,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15252,12 +15916,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15266,7 +15930,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15291,12 +15955,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB54_1
+; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -15305,7 +15969,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15333,12 +15997,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB54_1
+; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -15354,7 +16018,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15376,12 +16040,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB54_1
+; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -15397,7 +16061,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15420,18 +16084,18 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB54_1
+; GFX6-NEXT: s_cbranch_execnz .LBB58_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -15442,7 +16106,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15473,12 +16137,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-NEXT: s_cbranch_execnz .LBB59_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -15487,7 +16151,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15514,12 +16178,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB55_1
+; GFX940-NEXT: s_cbranch_execnz .LBB59_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX11-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
@@ -15528,7 +16192,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15560,20 +16224,20 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15600,12 +16264,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB55_1
+; GFX10-NEXT: s_cbranch_execnz .LBB59_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -15614,7 +16278,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15639,12 +16303,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
@@ -15653,7 +16317,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15678,12 +16342,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB55_1
+; GFX908-NEXT: s_cbranch_execnz .LBB59_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
@@ -15692,7 +16356,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -15720,12 +16384,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB55_1
+; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX7-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
@@ -15745,7 +16409,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15767,12 +16431,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB55_1
+; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg:
+; GFX6-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
@@ -15792,7 +16456,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -15815,18 +16479,18 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB55_1
+; GFX6-NEXT: s_cbranch_execnz .LBB59_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
- %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -15837,7 +16501,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, v3
@@ -15869,13 +16533,13 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_mov_b32_e32 v0, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -15884,7 +16548,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v7, v3
@@ -15911,13 +16575,13 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB56_1
+; GFX940-NEXT: s_cbranch_execnz .LBB60_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v0, v3
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -15926,7 +16590,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
@@ -15959,21 +16623,21 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
@@ -16000,13 +16664,13 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB56_1
+; GFX10-NEXT: s_cbranch_execnz .LBB60_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16015,7 +16679,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
@@ -16042,13 +16706,13 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16057,7 +16721,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
@@ -16082,13 +16746,13 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB56_1
+; GFX908-NEXT: s_cbranch_execnz .LBB60_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
@@ -16097,7 +16761,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
@@ -16125,12 +16789,12 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB56_1
+; GFX8-NEXT: s_cbranch_execnz .LBB60_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -16146,7 +16810,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -16168,14 +16832,14 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB56_1
+; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -16191,7 +16855,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
@@ -16214,7 +16878,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB56_1
+; GFX6-NEXT: s_cbranch_execnz .LBB60_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, v3
@@ -16222,12 +16886,12 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
+ %result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
}
-define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
+; GFX12-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
@@ -16238,7 +16902,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16269,12 +16933,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-NEXT: s_cbranch_execnz .LBB61_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
-; GFX940-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX940-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16283,7 +16947,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX940-NEXT: s_movk_i32 s4, 0x7fff
; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX940-NEXT: s_mov_b32 s5, 0x7060302
-; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX940-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16310,12 +16974,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX940-NEXT: v_mov_b32_e32 v3, v2
; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX940-NEXT: s_cbranch_execnz .LBB57_1
+; GFX940-NEXT: s_cbranch_execnz .LBB61_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX11-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
@@ -16324,7 +16988,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16356,20 +17020,20 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX10-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16396,12 +17060,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_cbranch_execnz .LBB57_1
+; GFX10-NEXT: s_cbranch_execnz .LBB61_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX90A-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX90A-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16410,7 +17074,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16437,12 +17101,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
+; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
-; GFX908-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX908-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
@@ -16451,7 +17115,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
-; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16476,12 +17140,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB57_1
+; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX8-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
@@ -16490,7 +17154,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -16518,12 +17182,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB57_1
+; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX7-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
@@ -16539,7 +17203,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -16561,12 +17225,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX7-NEXT: s_cbranch_execnz .LBB57_1
+; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX6-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos:
+; GFX6-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
@@ -16582,7 +17246,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX6-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
@@ -16605,15 +17269,17 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB57_1
+; GFX6-NEXT: s_cbranch_execnz .LBB61_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
- %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
+
+!0 = !{}
More information about the llvm-commits
mailing list