[llvm] [AMDGPU] Support atomic load and store for vector float types (v2f16, v2i16, v4i16, v4f16, v2f32) (PR #192904)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Sat May 2 06:38:23 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/192904
>From b503269a97c17b2a10a252d7c7aee95a38bf49a0 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 21 Apr 2026 18:01:07 +0800
Subject: [PATCH 1/4] [AMDGPU] Support atomic load and store for vector float
types (v2f16, v4f16, v2f32)
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 6 +
llvm/lib/Target/AMDGPU/FLATInstructions.td | 8 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +
.../inst-select-load-atomic-flat.mir | 60 +-
.../inst-select-load-atomic-global.mir | 48 +-
.../inst-select-store-atomic-flat.mir | 24 +-
llvm/test/CodeGen/AMDGPU/atomic-load-store.ll | 526 ++++++++++++++++++
7 files changed, 618 insertions(+), 66 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 0a4e61d1320c5..f92357e80282c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -168,6 +168,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f32, MVT::i64);
+
setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
@@ -180,6 +183,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f32, MVT::i64);
+
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c0fb73df9c764..a0963fcd5ce55 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2101,7 +2101,9 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
}
defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
@@ -2126,8 +2128,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
+defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2281,8 +2285,10 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
// the memory legalizer will set the cache bits and insert the
// appropriate waits.
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
@@ -2328,7 +2334,9 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8bb072a9626a9..0ada40036e7a5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -703,6 +703,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
+
setOperationAction(ISD::AND, MVT::v2i16, Promote);
AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
setOperationAction(ISD::OR, MVT::v2i16, Promote);
@@ -717,6 +723,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
+
setOperationAction(ISD::STORE, MVT::v4i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
setOperationAction(ISD::STORE, MVT::v4f16, Promote);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
index 5bfb2b2e4d578..eebf5fac2d9d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
@@ -70,37 +70,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX9-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX11-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX11-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX12-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 0)
$vgpr0 = COPY %1
@@ -274,37 +274,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX11-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX12-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 0)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
index 474f1308d8e24..3a79370b750ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
@@ -89,30 +89,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 1)
$vgpr0 = COPY %1
@@ -303,30 +303,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 1)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
index ae010a872a41d..3feb2698fed4a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
@@ -49,16 +49,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1_vgpr2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX7-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX7-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1_vgpr2
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX9-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX9-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
%0:vgpr(<2 x s16>) = COPY $vgpr0
%1:vgpr(p0) = COPY $vgpr1_vgpr2
G_STORE %0, %1 :: (store seq_cst (<2 x s16>), align 4, addrspace 0)
@@ -229,16 +229,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX7-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX7-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
%0:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
%1:vgpr(p0) = COPY $vgpr2_vgpr3
G_STORE %0, %1 :: (store seq_cst (<4 x s16>), align 8, addrspace 0)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
new file mode 100644
index 0000000000000..4c453f097eeca
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
@@ -0,0 +1,526 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+
+define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
>From 1e36617696ba94068ca425bece8a68b981db54b1 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Wed, 22 Apr 2026 18:34:15 +0800
Subject: [PATCH 2/4] [AMDGPU] Add v2i16 and v4i16 test
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +
llvm/test/CodeGen/AMDGPU/atomic-load-store.ll | 356 ++++++++++++++++++
2 files changed, 364 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0ada40036e7a5..cb3a5a4817312 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -703,9 +703,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2i16, MVT::i32);
setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2i16, MVT::i32);
setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
@@ -723,9 +727,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4i16, MVT::i64);
setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4i16, MVT::i64);
setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
index 4c453f097eeca..ad1e8c1cc0d53 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
@@ -318,6 +318,168 @@ define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, p
ret void
}
+define amdgpu_cs void @atomic_load_i16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -524,3 +686,197 @@ define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, p
store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
ret void
}
+
+define amdgpu_cs void @atomic_load_i16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
>From 298ae482c65ed5474d02eabdeee6b39e24da0428 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 23 Apr 2026 17:13:55 +0800
Subject: [PATCH 3/4] Update tests.
---
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 150 ++++++
...ic-load-store.ll => load-atomic-global.ll} | 448 +-----------------
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 73 +++
.../CodeGen/AMDGPU/store-atomic-global.ll | 213 +++++++++
4 files changed, 446 insertions(+), 438 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
rename llvm/test/CodeGen/AMDGPU/{atomic-load-store.ll => load-atomic-global.ll} (52%)
create mode 100644 llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
new file mode 100644
index 0000000000000..461fc93bece1d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
similarity index 52%
rename from llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
rename to llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index ad1e8c1cc0d53..962cc473de0e2 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -26,23 +26,25 @@ define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
; SDAG-NEXT: global_store_b32 v[2:3], v0, off
; SDAG-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
; GISEL-NEXT: global_store_b32 v[2:3], v0, off
; GISEL-NEXT: s_endpgm
- %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <2 x float> %a0, i32 0
%num2 = extractelement <2 x float> %a0, i32 1
%res = fadd float %num1, %num2
@@ -74,88 +76,6 @@ define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
- %num1 = extractelement <2 x float> %a0, i32 0
- %num2 = extractelement <2 x float> %a0, i32 1
- %res = fadd float %num1, %num2
- store float %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
- ret void
-}
-
define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x2_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -236,88 +156,6 @@ define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
- %num1 = extractelement <2 x half> %a0, i32 0
- %num2 = extractelement <2 x half> %a0, i32 1
- %res = fadd half %num1, %num2
- store half %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
define amdgpu_cs void @atomic_load_i16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_i16x2_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -398,88 +236,6 @@ define amdgpu_cs void @atomic_load_i16x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
- %e0 = extractelement <2 x i16> %a, i32 0
- %e1 = extractelement <2 x i16> %a, i32 1
- %sum = add i16 %e0, %e1
- store i16 %sum, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -517,7 +273,6 @@ define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1)
ret void
}
-
define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x4_global_seq_cst_agent:
; SDAG: ; %bb.0:
@@ -594,99 +349,6 @@ define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
- %num1 = extractelement <4 x half> %a0, i32 0
- %num2 = extractelement <4 x half> %a0, i32 1
- %num3 = extractelement <4 x half> %a0, i32 2
- %num4 = extractelement <4 x half> %a0, i32 3
- %add = fadd half %num1, %num2
- %mul = fmul half %num3, %num4
- %res = fadd half %add, %mul
- store half %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
define amdgpu_cs void @atomic_load_i16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_i16x4_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -790,93 +452,3 @@ define amdgpu_cs void @atomic_load_i16x4_global_monotonic_wavefront(ptr addrspac
store i16 %res, ptr addrspace(1) %out, align 4
ret void
}
-
-define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
- %num1 = extractelement <4 x i16> %a0, i32 0
- %num2 = extractelement <4 x i16> %a0, i32 1
- %num3 = extractelement <4 x i16> %a0, i32 2
- %num4 = extractelement <4 x i16> %a0, i32 3
- %add = add i16 %num1, %num2
- %mul = mul i16 %num3, %num4
- %res = add i16 %add, %mul
- store i16 %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
- ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
new file mode 100644
index 0000000000000..23982b79951e7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
new file mode 100644
index 0000000000000..87c2850e48c45
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
>From eff53c06265438d64f55dc5933d000e03dfe8936 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Sat, 2 May 2026 21:38:00 +0800
Subject: [PATCH 4/4] Update test and support lds vector atomic
---
llvm/lib/Target/AMDGPU/DSInstructions.td | 6 +
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 613 +++++--
.../test/CodeGen/AMDGPU/load-atomic-global.ll | 1410 +++++++++++++----
llvm/test/CodeGen/AMDGPU/load-atomic-local.ll | 1204 ++++++++++++++
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 229 ++-
.../CodeGen/AMDGPU/store-atomic-global.ll | 503 ++++--
.../test/CodeGen/AMDGPU/store-atomic-local.ll | 429 +++++
7 files changed, 3793 insertions(+), 601 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 48a8d2325af33..204cd89d4aefb 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -1004,7 +1004,10 @@ defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;
defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;
defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_nonext_32_local">;
+defm : DSReadPat_mc <DS_READ_B32, v2i16, "atomic_load_nonext_32_local">;
defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_nonext_64_local">;
+defm : DSReadPat_mc <DS_READ_B64, v2i32, "atomic_load_nonext_64_local">;
+defm : DSReadPat_mc <DS_READ_B64, v4i16, "atomic_load_nonext_64_local">;
let OtherPredicates = [D16PreservesUnusedBits] in {
// TODO: Atomic loads
@@ -1067,7 +1070,10 @@ defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;
defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
+defm : DSWritePat_mc <DS_WRITE_B32, v2i16, "atomic_store_32_local">;
defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
+defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "atomic_store_64_local">;
+defm : DSWritePat_mc <DS_WRITE_B64, v4i16, "atomic_store_64_local">;
let OtherPredicates = [HasD16LoadStore] in {
def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
index 461fc93bece1d..6a9062939d778 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -1,23 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
-define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <2 x float> %a0, i32 0
%num2 = extractelement <2 x float> %a0, i32 1
@@ -26,24 +48,58 @@ define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %
ret void
}
-define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -52,24 +108,59 @@ define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %
ret void
}
-define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -78,31 +169,76 @@ define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %
ret void
}
-define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -115,28 +251,71 @@ define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %
ret void
}
-define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -148,3 +327,235 @@ define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %
store i16 %res, ptr addrspace(1) %out, align 4
ret void
}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_min(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_min:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 -4096
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] offset:4095 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 4095
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_min(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 -4096
+ %a = load atomic <2 x i16>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dword v0, v[0:1] offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 4095
+ %a = load atomic <2 x i16>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index 962cc473de0e2..7e4b3e010c45e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -1,23 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
-define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f32x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <2 x float> %a0, i32 0
%num2 = extractelement <2 x float> %a0, i32 1
@@ -26,24 +48,44 @@ define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <2 x float> %a0, i32 0
%num2 = extractelement <2 x float> %a0, i32 1
@@ -52,22 +94,38 @@ define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %
ret void
}
-define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <2 x float> %a0, i32 0
%num2 = extractelement <2 x float> %a0, i32 1
@@ -76,24 +134,58 @@ define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -102,26 +194,67 @@ define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_f16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -130,24 +263,58 @@ define amdgpu_cs void @atomic_load_f16x2_global_seq_cst_agent(ptr addrspace(1) %
ret void
}
-define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -156,24 +323,59 @@ define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_i16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -182,26 +384,68 @@ define amdgpu_cs void @atomic_load_i16x2_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_i16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -210,24 +454,59 @@ define amdgpu_cs void @atomic_load_i16x2_global_seq_cst_agent(ptr addrspace(1) %
ret void
}
-define amdgpu_cs void @atomic_load_i16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -236,31 +515,76 @@ define amdgpu_cs void @atomic_load_i16x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -273,33 +597,85 @@ define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -312,31 +688,76 @@ define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %
ret void
}
-define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -349,28 +770,71 @@ define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_i16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -383,30 +847,80 @@ define amdgpu_cs void @atomic_load_i16x4_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_i16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -419,28 +933,71 @@ define amdgpu_cs void @atomic_load_i16x4_global_seq_cst_agent(ptr addrspace(1) %
ret void
}
-define amdgpu_cs void @atomic_load_i16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v1, v4, v0
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v4
+; GFX10-NEXT: v_mad_u16 v0, v1, v5, v0
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -452,3 +1009,238 @@ define amdgpu_cs void @atomic_load_i16x4_global_monotonic_wavefront(ptr addrspac
store i16 %res, ptr addrspace(1) %out, align 4
ret void
}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:1 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:2047 glc dlc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:4095 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:1 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:1 glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
+ %a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4095 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: global_load_dword v0, v[0:1], off offset:2047 glc dlc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: global_store_short v[2:3], v0, off
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-GISEL-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-GISEL-NEXT: global_store_short v[2:3], v0, off
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
+ %a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
new file mode 100644
index 0000000000000..3bd6d043bab45
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
@@ -0,0 +1,1204 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v3, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_add_nc_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v0, v3, v4, v0
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v3, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_add_nc_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v0, v3, v4, v0
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v3, v0, v2
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_add_nc_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v0, v3, v4, v0
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0 offset:1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0 offset:1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0 offset:1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0 offset:1
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0 offset:4095
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0 offset:4095
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0 offset:4095
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0 offset:4095
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
+ %a0 = load atomic <2 x float>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0 offset:1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0 offset:1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
+ %a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b32 v0, v0 offset:4095
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: ds_write_b16 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b32 v0, v0 offset:4095
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: ds_write_b16 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
+ %a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(3) %out, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
index 23982b79951e7..c2f3128162ab8 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -1,73 +1,198 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
-define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
ret void
}
-define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dword v[1:2], v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dword v[1:2], v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b32 v[1:2], v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
ret void
}
-define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dword v[1:2], v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dword v[1:2], v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b32 v[1:2], v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
ret void
}
-define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
ret void
}
-define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
ret void
}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, 0xfffff000, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, -1, v3, vcc_lo
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %out, i64 -4096
+ store atomic <4 x i16> %in, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1] offset:4088
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0xff8, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] offset:4088
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:4088 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(0) %out, i64 4088
+ store atomic <4 x i16> %in, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
index 87c2850e48c45..60f8194ea698a 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -1,213 +1,438 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
-
-define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
ret void
}
-define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
ret void
}
-define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
ret void
}
-define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
ret void
}
-define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX12-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
ret void
}
-define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off
-; GISEL-NEXT: s_endpgm
+; GFX10-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off
+; GFX12-NEXT: s_endpgm
store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
ret void
}
-define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
ret void
}
-define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX12-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
ret void
}
-define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off
-; GISEL-NEXT: s_endpgm
+; GFX10-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b32 v[1:2], v0, off
+; GFX12-NEXT: s_endpgm
store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
ret void
}
-define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
ret void
}
-define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX11-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
ret void
}
-define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
+; GFX12-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
ret void
}
-define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
ret void
}
-define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
+; GFX11-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
ret void
}
-define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
+define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
+; GFX12-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
ret void
}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:-4096
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, -1, v3, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off offset:-4096
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off offset:-4096 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %out, i64 -4096
+ store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:4088
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v2, vcc_lo, 0x800, v2
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:2040
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xff8, v2
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off offset:4088
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off offset:4088 scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4088
+ store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
new file mode 100644
index 0000000000000..0d2f48d886f2d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
@@ -0,0 +1,429 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(3) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_1(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:1
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1] offset:1
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1] offset:1
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1] offset:1
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %out, i64 1
+ store atomic <4 x i16> %in, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:4088
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1] offset:4088
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1] offset:4088
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1] offset:4088
+; GFX12-NEXT: s_endpgm
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %out, i64 4088
+ store atomic <4 x i16> %in, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
+; GFX9-GISEL: {{.*}}
+; GFX9-SDAG: {{.*}}
More information about the llvm-commits
mailing list