[llvm] [AMDGPU] Support atomic load and store for vector float types (v2f16, v4f16, v2f32) (PR #192904)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 21 03:01:26 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/192904
>From b503269a97c17b2a10a252d7c7aee95a38bf49a0 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 21 Apr 2026 18:01:07 +0800
Subject: [PATCH] [AMDGPU] Support atomic load and store for vector float types
(v2f16, v4f16, v2f32)
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 6 +
llvm/lib/Target/AMDGPU/FLATInstructions.td | 8 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +
.../inst-select-load-atomic-flat.mir | 60 +-
.../inst-select-load-atomic-global.mir | 48 +-
.../inst-select-store-atomic-flat.mir | 24 +-
llvm/test/CodeGen/AMDGPU/atomic-load-store.ll | 526 ++++++++++++++++++
7 files changed, 618 insertions(+), 66 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 0a4e61d1320c5..f92357e80282c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -168,6 +168,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f32, MVT::i64);
+
setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
@@ -180,6 +183,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f32, MVT::i64);
+
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c0fb73df9c764..a0963fcd5ce55 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2101,7 +2101,9 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
}
defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
@@ -2126,8 +2128,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
+defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2281,8 +2285,10 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
// the memory legalizer will set the cache bits and insert the
// appropriate waits.
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
@@ -2328,7 +2334,9 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8bb072a9626a9..0ada40036e7a5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -703,6 +703,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
+
setOperationAction(ISD::AND, MVT::v2i16, Promote);
AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
setOperationAction(ISD::OR, MVT::v2i16, Promote);
@@ -717,6 +723,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
+ setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
+
setOperationAction(ISD::STORE, MVT::v4i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
setOperationAction(ISD::STORE, MVT::v4f16, Promote);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
index 5bfb2b2e4d578..eebf5fac2d9d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
@@ -70,37 +70,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX9-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX11-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX11-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX12-LABEL: name: load_atomic_flat_v2s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>))
- ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>))
+ ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 0)
$vgpr0 = COPY %1
@@ -274,37 +274,37 @@ body: |
; GFX7-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX11-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX11: liveins: $vgpr0_vgpr1
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX12-LABEL: name: load_atomic_flat_v4s16_seq_cst
; GFX12: liveins: $vgpr0_vgpr1
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>))
- ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX12-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>))
+ ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
%0:vgpr(p0) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 0)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
index 474f1308d8e24..3a79370b750ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
@@ -89,30 +89,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
;
; GFX9-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
;
; GFX10-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 1)
$vgpr0 = COPY %1
@@ -303,30 +303,30 @@ body: |
; GFX7-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
; GFX7-FLAT-NEXT: {{ $}}
- ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
;
; GFX9-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
;
; GFX10-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX10: liveins: $vgpr0_vgpr1
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
%0:vgpr(p1) = COPY $vgpr0_vgpr1
%1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 1)
$vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
index ae010a872a41d..3feb2698fed4a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir
@@ -49,16 +49,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX7: liveins: $vgpr0, $vgpr1_vgpr2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX7-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX7-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v2s16_seq_cst
; GFX9: liveins: $vgpr0, $vgpr1_vgpr2
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2
- ; GFX9-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+ ; GFX9-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s16>))
%0:vgpr(<2 x s16>) = COPY $vgpr0
%1:vgpr(p0) = COPY $vgpr1_vgpr2
G_STORE %0, %1 :: (store seq_cst (<2 x s16>), align 4, addrspace 0)
@@ -229,16 +229,16 @@ body: |
; GFX7-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX7-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX7-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
;
; GFX9-LABEL: name: atomic_store_flat_v4s16_seq_cst
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>))
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<4 x s16>))
%0:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
%1:vgpr(p0) = COPY $vgpr2_vgpr3
G_STORE %0, %1 :: (store seq_cst (<4 x s16>), align 8, addrspace 0)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
new file mode 100644
index 0000000000000..4c453f097eeca
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/atomic-load-store.ll
@@ -0,0 +1,526 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s
+
+define amdgpu_cs void @atomic_load_f32x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_flat_monotonic_agent(<2 x float> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_global_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f32x2_global_seq_cst_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x2_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x half> %a0, i32 0
+ %num2 = extractelement <2 x half> %a0, i32 1
+ %res = fadd half %num1, %num2
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_global_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b32 v[1:2], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b32 v[1:2], v0, off
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x2_flat_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x2_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b32 v[1:2], v0 scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <2 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_global_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+
+define amdgpu_cs void @atomic_load_f16x4_global_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: global_inv scope:SCOPE_DEV
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_global_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f16x4_flat_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_load_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GISEL-NEXT: s_endpgm
+ %a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <4 x half> %a0, i32 0
+ %num2 = extractelement <4 x half> %a0, i32 1
+ %num3 = extractelement <4 x half> %a0, i32 2
+ %num4 = extractelement <4 x half> %a0, i32 3
+ %add = fadd half %num1, %num2
+ %mul = fmul half %num3, %num4
+ %res = fadd half %add, %mul
+ store half %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_seq_cst_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_global_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; SDAG-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_global_monotonic_wavefront:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f16x4_flat_monotonic_agent(<4 x half> %in, ptr addrspace(0) %out) {
+; SDAG-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: atomic_store_f16x4_flat_monotonic_agent:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GISEL-NEXT: s_endpgm
+ store atomic <4 x half> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
More information about the llvm-commits
mailing list