[llvm] e4d5880 - [AMDGPU] Support atomic load and store for vector float types (v2f16, v2i16, v4i16, v4f16, v2f32) (#192904)
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 20:18:19 PDT 2026
Author: Harrison Hao
Date: 2026-05-09T11:18:13+08:00
New Revision: e4d58800d9990d93d57e32c2c6d9895a466c8c82
URL: https://github.com/llvm/llvm-project/commit/e4d58800d9990d93d57e32c2c6d9895a466c8c82
DIFF: https://github.com/llvm/llvm-project/commit/e4d58800d9990d93d57e32c2c6d9895a466c8c82.diff
LOG: [AMDGPU] Support atomic load and store for vector float types (v2f16, v2i16, v4i16, v4f16, v2f32) (#192904)
Add support for atomic load and store on <2 x half>, <4 x half>, and
<2 x float> vector types in the AMDGPU backend.
These types are promoted to equivalently sized integer types before
instruction selection:
<2 x half> -> i32
<4 x half> -> i64
<2 x i16> -> i32
<4 x i16> -> i64
<2 x float> -> i64
Added:
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
llvm/lib/Target/AMDGPU/DSInstructions.td
llvm/lib/Target/AMDGPU/FLATInstructions.td
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-local.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-local.mir
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index ee41e5bf73ae2..058ca9c29625f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -168,6 +168,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f32, MVT::i64);
+
setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
@@ -180,6 +183,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f32, MVT::i64);
+
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 48a8d2325af33..204cd89d4aefb 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -1004,7 +1004,10 @@ defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;
defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;
defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_nonext_32_local">;
+defm : DSReadPat_mc <DS_READ_B32, v2i16, "atomic_load_nonext_32_local">;
defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_nonext_64_local">;
+defm : DSReadPat_mc <DS_READ_B64, v2i32, "atomic_load_nonext_64_local">;
+defm : DSReadPat_mc <DS_READ_B64, v4i16, "atomic_load_nonext_64_local">;
let OtherPredicates = [D16PreservesUnusedBits] in {
// TODO: Atomic loads
@@ -1067,7 +1070,10 @@ defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;
defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
+defm : DSWritePat_mc <DS_WRITE_B32, v2i16, "atomic_store_32_local">;
defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
+defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "atomic_store_64_local">;
+defm : DSWritePat_mc <DS_WRITE_B64, v4i16, "atomic_store_64_local">;
let OtherPredicates = [HasD16LoadStore] in {
def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c0fb73df9c764..a0963fcd5ce55 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2101,7 +2101,9 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
}
defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
@@ -2126,8 +2128,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
+defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2281,8 +2285,10 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
// the memory legalizer will set the cache bits and insert the
// appropriate waits.
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
@@ -2328,7 +2334,9 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 20599228beea8..95b214f0da4c8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -705,6 +705,16 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2i16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2i16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
+
setOperationAction(ISD::AND, MVT::v2i16, Promote);
AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
setOperationAction(ISD::OR, MVT::v2i16, Promote);
@@ -719,6 +729,16 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4i16, MVT::i64);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4i16, MVT::i64);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
+
setOperationAction(ISD::STORE, MVT::v4i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
setOperationAction(ISD::STORE, MVT::v4f16, Promote);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
index 5bfb2b2e4d578..eebf5fac2d9d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
@@ -70,37 +70,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX9-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX11-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX11-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX12-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 0)
$vgpr0 = COPY %1
@@ -274,37 +274,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX11-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX12-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 0)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
index 474f1308d8e24..3a79370b750ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
@@ -89,30 +89,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 1)
$vgpr0 = COPY %1
@@ -303,30 +303,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 1)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-local.mir
index 5c2df3904b817..ffaa84d3ca700 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-local.mir
@@ -24,6 +24,7 @@ body: |
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (s32), addrspace 3)
; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+ ;
; GFX7-LABEL: name: load_atomic_local_s32_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
@@ -31,6 +32,7 @@ body: |
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (s32), addrspace 3)
; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+ ;
; GFX9-LABEL: name: load_atomic_local_s32_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
@@ -57,23 +59,25 @@ body: |
; GFX6-LABEL: name: load_atomic_local_v2s16_seq_cst
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<2 x s16>), addrspace 3)
- ; GFX6-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 3)
+ ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+ ;
; GFX7-LABEL: name: load_atomic_local_v2s16_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7-NEXT: $m0 = S_MOV_B32 -1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<2 x s16>), addrspace 3)
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 3)
+ ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+ ;
; GFX9-LABEL: name: load_atomic_local_v2s16_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<2 x s16>), addrspace 3)
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 3)
+ ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 3)
$vgpr0 = COPY %1
@@ -98,6 +102,7 @@ body: |
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load seq_cst (p3), addrspace 3)
; GFX6-NEXT: $vgpr0 = COPY [[LOAD]](p3)
+ ;
; GFX7-LABEL: name: load_atomic_local_p3_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
@@ -105,6 +110,7 @@ body: |
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load seq_cst (p3), addrspace 3)
; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](p3)
+ ;
; GFX9-LABEL: name: load_atomic_local_p3_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
@@ -135,6 +141,7 @@ body: |
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (s64), addrspace 3)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+ ;
; GFX7-LABEL: name: load_atomic_local_s64_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
@@ -142,6 +149,7 @@ body: |
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (s64), addrspace 3)
; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+ ;
; GFX9-LABEL: name: load_atomic_local_s64_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
@@ -168,23 +176,25 @@ body: |
; GFX6-LABEL: name: load_atomic_local_v2s32_seq_cst
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<2 x s32>), addrspace 3)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+ ; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 3)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+ ;
; GFX7-LABEL: name: load_atomic_local_v2s32_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7-NEXT: $m0 = S_MOV_B32 -1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<2 x s32>), addrspace 3)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+ ; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 3)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+ ;
; GFX9-LABEL: name: load_atomic_local_v2s32_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<2 x s32>), addrspace 3)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 3)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0
%1:vgpr(<2 x s32>) = G_LOAD %0 :: (load seq_cst (<2 x s32>), align 8, addrspace 3)
$vgpr0_vgpr1 = COPY %1
@@ -205,23 +215,25 @@ body: |
; GFX6-LABEL: name: load_atomic_local_v4s16_seq_cst
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<4 x s16>), addrspace 3)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 3)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+ ;
; GFX7-LABEL: name: load_atomic_local_v4s16_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7-NEXT: $m0 = S_MOV_B32 -1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<4 x s16>), addrspace 3)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 3)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+ ;
; GFX9-LABEL: name: load_atomic_local_v4s16_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load seq_cst (<4 x s16>), addrspace 3)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 3)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 3)
$vgpr0_vgpr1 = COPY %1
@@ -246,6 +258,7 @@ body: |
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p1) = G_LOAD [[COPY]](p3) :: (load seq_cst (p1), addrspace 3)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+ ;
; GFX7-LABEL: name: load_atomic_local_p1_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
@@ -253,6 +266,7 @@ body: |
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p1) = G_LOAD [[COPY]](p3) :: (load seq_cst (p1), addrspace 3)
; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+ ;
; GFX9-LABEL: name: load_atomic_local_p1_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
@@ -283,6 +297,7 @@ body: |
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p0) = G_LOAD [[COPY]](p3) :: (load seq_cst (p0), addrspace 3)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p0)
+ ;
; GFX7-LABEL: name: load_atomic_local_p0_seq_cst
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
@@ -290,6 +305,7 @@ body: |
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p0) = G_LOAD [[COPY]](p3) :: (load seq_cst (p0), addrspace 3)
; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p0)
+ ;
; GFX9-LABEL: name: load_atomic_local_p0_seq_cst
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
@@ -322,6 +338,7 @@ body: |
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[V_ADD_CO_U32_e64_]], 0, 0, implicit $m0, implicit $exec :: (load seq_cst (s32), addrspace 3)
; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+ ;
; GFX7-LABEL: name: load_atomic_local_s32_seq_cst_gep_65535
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
@@ -329,6 +346,7 @@ body: |
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 65535, 0, implicit $m0, implicit $exec :: (load seq_cst (s32), addrspace 3)
; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+ ;
; GFX9-LABEL: name: load_atomic_local_s32_seq_cst_gep_65535
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
index ae010a872a41d..3feb2698fed4a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
@@ -49,16 +49,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1_vgpr2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX7-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX7-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1_vgpr2
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX9-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX9-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
%0:vgpr(<2 x s16>) = COPY $vgpr0
%1:vgpr(p0) = COPY $vgpr1_vgpr2
G_STORE %0, %1 :: (store seq_cst (<2 x s16>), align 4, addrspace 0)
@@ -229,16 +229,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX7-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX7-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
%0:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
%1:vgpr(p0) = COPY $vgpr2_vgpr3
G_STORE %0, %1 :: (store seq_cst (<4 x s16>), align 8, addrspace 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-local.mir
index d290f1b2403e4..2e39861060716 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-local.mir
@@ -24,6 +24,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (s32), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_s32_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1
; GFX7-NEXT: {{ $}}
@@ -31,6 +32,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (s32), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_s32_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -57,23 +59,25 @@ body: |
; GFX6-LABEL: name: atomic_store_local_v2s16_seq_cst
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p3) :: (store seq_cst (<2 x s16>), addrspace 3)
+ ; GFX6-NEXT: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (<2 x s16>), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_v2s16_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX7-NEXT: $m0 = S_MOV_B32 -1
- ; GFX7-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p3) :: (store seq_cst (<2 x s16>), addrspace 3)
+ ; GFX7-NEXT: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (<2 x s16>), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_v2s16_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
- ; GFX9-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p3) :: (store seq_cst (<2 x s16>), addrspace 3)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: DS_WRITE_B32_gfx9 [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (store seq_cst (<2 x s16>), addrspace 3)
%0:vgpr(<2 x s16>) = COPY $vgpr0
%1:vgpr(p3) = COPY $vgpr1
G_STORE %0, %1 :: (store seq_cst (<2 x s16>), align 4, addrspace 3)
@@ -98,6 +102,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: G_STORE [[COPY]](p3), [[COPY1]](p3) :: (store seq_cst (p3), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_p3_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1
; GFX7-NEXT: {{ $}}
@@ -105,6 +110,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: G_STORE [[COPY]](p3), [[COPY1]](p3) :: (store seq_cst (p3), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_p3_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -135,6 +141,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: G_STORE [[COPY]](p5), [[COPY1]](p3) :: (store seq_cst (p5), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_p5_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1
; GFX7-NEXT: {{ $}}
@@ -142,6 +149,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: G_STORE [[COPY]](p5), [[COPY1]](p3) :: (store seq_cst (p5), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_p5_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -172,6 +180,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: G_STORE [[COPY]](p6), [[COPY1]](p3) :: (store seq_cst (p6), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_p6_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1
; GFX7-NEXT: {{ $}}
@@ -179,6 +188,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: G_STORE [[COPY]](p6), [[COPY1]](p3) :: (store seq_cst (p6), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_p6_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -209,6 +219,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: DS_WRITE_B64 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (s64), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_s64_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -216,6 +227,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: DS_WRITE_B64 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (s64), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_s64_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2
; GFX9-NEXT: {{ $}}
@@ -242,23 +254,25 @@ body: |
; GFX6-LABEL: name: atomic_store_local_v2s32_seq_cst
; GFX6: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: G_STORE [[COPY]](<2 x s32>), [[COPY1]](p3) :: (store seq_cst (<2 x s32>), addrspace 3)
+ ; GFX6-NEXT: DS_WRITE_B64 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (<2 x s32>), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_v2s32_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX7-NEXT: $m0 = S_MOV_B32 -1
- ; GFX7-NEXT: G_STORE [[COPY]](<2 x s32>), [[COPY1]](p3) :: (store seq_cst (<2 x s32>), addrspace 3)
+ ; GFX7-NEXT: DS_WRITE_B64 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (<2 x s32>), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_v2s32_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
- ; GFX9-NEXT: G_STORE [[COPY]](<2 x s32>), [[COPY1]](p3) :: (store seq_cst (<2 x s32>), addrspace 3)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX9-NEXT: DS_WRITE_B64_gfx9 [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (store seq_cst (<2 x s32>), addrspace 3)
%0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
%1:vgpr(p3) = COPY $vgpr2
G_STORE %0, %1 :: (store seq_cst (<2 x s32>), align 8, addrspace 3)
@@ -279,23 +293,25 @@ body: |
; GFX6-LABEL: name: atomic_store_local_v4s16_seq_cst
; GFX6: liveins: $vgpr0_vgpr1, $vgpr2
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p3) :: (store seq_cst (<4 x s16>), addrspace 3)
+ ; GFX6-NEXT: DS_WRITE_B64 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (<4 x s16>), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX7-NEXT: $m0 = S_MOV_B32 -1
- ; GFX7-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p3) :: (store seq_cst (<4 x s16>), addrspace 3)
+ ; GFX7-NEXT: DS_WRITE_B64 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store seq_cst (<4 x s16>), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
- ; GFX9-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p3) :: (store seq_cst (<4 x s16>), addrspace 3)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX9-NEXT: DS_WRITE_B64_gfx9 [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (store seq_cst (<4 x s16>), addrspace 3)
%0:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
%1:vgpr(p3) = COPY $vgpr2
G_STORE %0, %1 :: (store seq_cst (<4 x s16>), align 8, addrspace 3)
@@ -320,6 +336,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: G_STORE [[COPY]](p0), [[COPY1]](p3) :: (store seq_cst (p0), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_p0_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -327,6 +344,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: G_STORE [[COPY]](p0), [[COPY1]](p3) :: (store seq_cst (p0), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_p0_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2
; GFX9-NEXT: {{ $}}
@@ -356,6 +374,7 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
; GFX6-NEXT: $m0 = S_MOV_B32 -1
; GFX6-NEXT: G_STORE [[COPY]](p1), [[COPY1]](p3) :: (store seq_cst (p1), addrspace 3)
+ ;
; GFX7-LABEL: name: atomic_store_local_p1_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -363,6 +382,7 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr2
; GFX7-NEXT: $m0 = S_MOV_B32 -1
; GFX7-NEXT: G_STORE [[COPY]](p1), [[COPY1]](p3) :: (store seq_cst (p1), addrspace 3)
+ ;
; GFX9-LABEL: name: atomic_store_local_p1_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2
; GFX9-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
new file mode 100644
index 0000000000000..6a9062939d778
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -0,0 +1,561 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_min(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 -4096
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] offset:4095 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 4095
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_min(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 -4096
+ %a = load atomic <2 x i16>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dword v0, v[0:1] offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 4095
+ %a = load atomic <2 x i16>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
new file mode 100644
index 0000000000000..7e4b3e010c45e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -0,0 +1,1246 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:1 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:2047 glc dlc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:4095 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:1 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:1 glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
+ %a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: global_load_dword v0, v[0:1], off offset:2047 glc dlc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: global_store_short v[2:3], v0, off
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-GISEL-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-GISEL-NEXT: global_store_short v[2:3], v0, off
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
+ %a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
new file mode 100644
index 0000000000000..3bd6d043bab45
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
@@ -0,0 +1,1204 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v3, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_add_nc_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v0, v3, v4, v0
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v3, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_add_nc_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v0, v3, v4, v0
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v3, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_add_nc_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v0, v3, v4, v0
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0 offset:1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0 offset:1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0 offset:1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0 offset:1
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0 offset:4095
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0 offset:4095
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0 offset:4095
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0 offset:4095
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0 offset:1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0 offset:1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
+ %a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0 offset:4095
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0 offset:4095
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
+ %a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
new file mode 100644
index 0000000000000..c2f3128162ab8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dword v[1:2], v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dword v[1:2], v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b32 v[1:2], v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dword v[1:2], v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dword v[1:2], v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b32 v[1:2], v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, 0xfffff000, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, -1, v3, vcc_lo
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %out, i64 -4096
+ store atomic <4 x i16> %in, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1] offset:4088
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0xff8, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] offset:4088
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:4088 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %out, i64 4088
+ store atomic <4 x i16> %in, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
new file mode 100644
index 0000000000000..60f8194ea698a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -0,0 +1,438 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:-4096
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, -1, v3, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off offset:-4096
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off offset:-4096 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %out, i64 -4096
+ store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:4088
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v2, vcc_lo, 0x800, v2
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:2040
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xff8, v2
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off offset:4088
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off offset:4088 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4088
+ store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
new file mode 100644
index 0000000000000..0d2f48d886f2d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
@@ -0,0 +1,429 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(3) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_1(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:1
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1] offset:1
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1] offset:1
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1] offset:1
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %out, i64 1
+ store atomic <4 x i16> %in, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:4088
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1] offset:4088
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1] offset:4088
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1] offset:4088
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %out, i64 4088
+ store atomic <4 x i16> %in, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
More information about the llvm-commits
mailing list