[llvm] [AMDGPU] Support atomic load and store for vector float types (v2f16, v2i6, v4i16, v4f16, v2f32) (PR #192904)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 02:14:18 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/192904
>From b503269a97c17b2a10a252d7c7aee95a38bf49a0 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 21 Apr 2026 18:01:07 +0800
Subject: [PATCH 1/3] [AMDGPU] Support atomic load and store for vector float
types (v2f16, v4f16, v2f32)
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 6 +
llvm/lib/Target/AMDGPU/FLATInstructions.td | 8 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +
.../inst-select-load-atomic-flat.mir | 60 +-
.../inst-select-load-atomic-global.mir | 48 +-
.../inst-select-store-atomic-flat.mir | 24 +-
llvm/test/CodeGen/AMDGPU/atomic-load-store.ll | 526 ++++++++++++++++++
7 files changed, 618 insertions(+), 66 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 0a4e61d1320c5..f92357e80282c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -168,6 +168,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f32, MVT::i64);
+
setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
@@ -180,6 +183,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f32, MVT::i64);
+
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c0fb73df9c764..a0963fcd5ce55 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2101,7 +2101,9 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
}
defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
@@ -2126,8 +2128,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
+defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2281,8 +2285,10 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
// the memory legalizer will set the cache bits and insert the
// appropriate waits.
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
@@ -2328,7 +2334,9 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8bb072a9626a9..0ada40036e7a5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -703,6 +703,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
+
setOperationAction(ISD::AND, MVT::v2i16, Promote);
AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
setOperationAction(ISD::OR, MVT::v2i16, Promote);
@@ -717,6 +723,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
+
setOperationAction(ISD::STORE, MVT::v4i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
setOperationAction(ISD::STORE, MVT::v4f16, Promote);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
index 5bfb2b2e4d578..eebf5fac2d9d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
@@ -70,37 +70,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX9-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX11-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX11-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX12-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 0)
$vgpr0 = COPY %1
@@ -274,37 +274,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX11-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX12-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 0)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
index 474f1308d8e24..3a79370b750ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
@@ -89,30 +89,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 1)
$vgpr0 = COPY %1
@@ -303,30 +303,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 1)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
index ae010a872a41d..3feb2698fed4a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
@@ -49,16 +49,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1_vgpr2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX7-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX7-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1_vgpr2
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX9-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX9-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
%0:vgpr(<2 x s16>) = COPY $vgpr0
%1:vgpr(p0) = COPY $vgpr1_vgpr2
G_STORE %0, %1 :: (store seq_cst (<2 x s16>), align 4, addrspace 0)
@@ -229,16 +229,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX7-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX7-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
%0:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
%1:vgpr(p0) = COPY $vgpr2_vgpr3
G_STORE %0, %1 :: (store seq_cst (<4 x s16>), align 8, addrspace 0)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
new file mode 100644
index 0000000000000..4c453f097eeca
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
@@ -0,0 +1,526 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+
+define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
>From 1e36617696ba94068ca425bece8a68b981db54b1 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Wed, 22 Apr 2026 18:34:15 +0800
Subject: [PATCH 2/3] [AMDGPU] Add v2i16 and v4i16 test
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +
llvm/test/CodeGen/AMDGPU/atomic-load-store.ll | 356 ++++++++++++++++++
2 files changed, 364 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0ada40036e7a5..cb3a5a4817312 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -703,9 +703,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2i16, MVT::i32);
setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2i16, MVT::i32);
setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
@@ -723,9 +727,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4i16, MVT::i64);
setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4i16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4i16, MVT::i64);
setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
index 4c453f097eeca..ad1e8c1cc0d53 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
@@ -318,6 +318,168 @@ define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, p
ret void
}
+define amdgpu_cs void @atomic_load_i16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -524,3 +686,197 @@ define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, p
store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
ret void
}
+
+define amdgpu_cs void @atomic_load_i16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
>From 298ae482c65ed5474d02eabdeee6b39e24da0428 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 23 Apr 2026 17:13:55 +0800
Subject: [PATCH 3/3] Update tests.
---
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 150 ++++++
...ic-load-store.ll => load-atomic-global.ll} | 448 +-----------------
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 73 +++
.../CodeGen/AMDGPU/store-atomic-global.ll | 213 +++++++++
4 files changed, 446 insertions(+), 438 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
rename llvm/test/CodeGen/AMDGPU/{atomic-load-store.ll => load-atomic-global.ll} (52%)
create mode 100644 llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
new file mode 100644
index 0000000000000..461fc93bece1d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %e0 = extractelement <2 x i16> %a, i32 0
+ %e1 = extractelement <2 x i16> %a, i32 1
+ %sum = add i16 %e0, %e1
+ store i16 %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x i16> %a0, i32 0
+ %num2 = extractelement <4 x i16> %a0, i32 1
+ %num3 = extractelement <4 x i16> %a0, i32 2
+ %num4 = extractelement <4 x i16> %a0, i32 3
+ %add = add i16 %num1, %num2
+ %mul = mul i16 %num3, %num4
+ %res = add i16 %add, %mul
+ store i16 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
similarity index 52%
rename from llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
rename to llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index ad1e8c1cc0d53..962cc473de0e2 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -26,23 +26,25 @@ define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1)
ret void
}
-define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
; SDAG-NEXT: global_store_b32 v[2:3], v0, off
; SDAG-NEXT: s_endpgm
;
-; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
; GISEL-NEXT: global_store_b32 v[2:3], v0, off
; GISEL-NEXT: s_endpgm
- %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <2 x float> %a0, i32 0
%num2 = extractelement <2 x float> %a0, i32 1
%res = fadd float %num1, %num2
@@ -74,88 +76,6 @@ define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: global_inv scope:SCOPE_DEV
-; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: global_inv scope:SCOPE_DEV
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
- %num1 = extractelement <2 x float> %a0, i32 0
- %num2 = extractelement <2 x float> %a0, i32 1
- %res = fadd float %num1, %num2
- store float %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
- store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
- ret void
-}
-
define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x2_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -236,88 +156,6 @@ define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
- %num1 = extractelement <2 x half> %a0, i32 0
- %num2 = extractelement <2 x half> %a0, i32 1
- %res = fadd half %num1, %num2
- store half %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
define amdgpu_cs void @atomic_load_i16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_i16x2_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -398,88 +236,6 @@ define amdgpu_cs void @atomic_load_i16x2_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_i16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
- %e0 = extractelement <2 x i16> %a, i32 0
- %e1 = extractelement <2 x i16> %a, i32 1
- %sum = add i16 %e0, %e1
- store i16 %sum, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b32 v[1:2], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b32 v[1:2], v0, off
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
- ret void
-}
-
define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -517,7 +273,6 @@ define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1)
ret void
}
-
define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_f16x4_global_seq_cst_agent:
; SDAG: ; %bb.0:
@@ -594,99 +349,6 @@ define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspac
ret void
}
-define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
- %num1 = extractelement <4 x half> %a0, i32 0
- %num2 = extractelement <4 x half> %a0, i32 1
- %num3 = extractelement <4 x half> %a0, i32 2
- %num4 = extractelement <4 x half> %a0, i32 3
- %add = fadd half %num1, %num2
- %mul = fmul half %num3, %num4
- %res = fadd half %add, %mul
- store half %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
define amdgpu_cs void @atomic_load_i16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; SDAG-LABEL: atomic_load_i16x4_global_monotonic_agent:
; SDAG: ; %bb.0:
@@ -790,93 +452,3 @@ define amdgpu_cs void @atomic_load_i16x4_global_monotonic_wavefront(ptr addrspac
store i16 %res, ptr addrspace(1) %out, align 4
ret void
}
-
-define amdgpu_cs void @atomic_load_i16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_load_i16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; SDAG-NEXT: global_store_b16 v[2:3], v0, off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_load_i16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
-; GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
-; GISEL-NEXT: global_store_b16 v[2:3], v0, off
-; GISEL-NEXT: s_endpgm
- %a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
- %num1 = extractelement <4 x i16> %a0, i32 0
- %num2 = extractelement <4 x i16> %a0, i32 1
- %num3 = extractelement <4 x i16> %a0, i32 2
- %num4 = extractelement <4 x i16> %a0, i32 3
- %add = add i16 %num1, %num2
- %mul = mul i16 %num3, %num4
- %res = add i16 %add, %mul
- store i16 %res, ptr addrspace(1) %out, align 4
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
-; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
- ret void
-}
-
-define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
-; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
-; GISEL-NEXT: s_endpgm
- store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
- ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
new file mode 100644
index 0000000000000..23982b79951e7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_flat_monotonic_agent(<2 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_flat_monotonic_agent(<4 x i16> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
new file mode 100644
index 0000000000000..87c2850e48c45
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x2_global_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_i16x4_global_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_i16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
More information about the llvm-commits
mailing list