[llvm-branch-commits] [llvm] AMDGPU: Use normal load/store pattern type lists for atomics (PR #229688)
Matt Arsenault via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 7 01:19:05 PDT 2026
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/229688
Atomic load and store of vector types are now permitted in the IR. Instead of
maintaining separate scalar-only atomic pattern lists, cover atomic load/store
in the existing per-register-type pattern loops. As a side effect
-flat-for-global is respected in more cases.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
>From 7716d9e6a1afbb0151a438e791b008d8fe93c38f Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 09:51:48 +0200
Subject: [PATCH] AMDGPU: Use normal load/store pattern type lists for atomics
Atomic load and store of vector types are now permitted in the IR. Instead of
maintaining separate scalar-only atomic pattern lists, cover atomic load/store
in the existing per-register-type pattern loops. As a side effect
-flat-for-global is respected in more cases.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 22 +-
llvm/lib/Target/AMDGPU/BUFInstructions.td | 22 +-
llvm/lib/Target/AMDGPU/DSInstructions.td | 20 +-
llvm/lib/Target/AMDGPU/FLATInstructions.td | 44 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 43 +-
.../inst-select-load-atomic-global.mir | 60 +-
llvm/test/CodeGen/AMDGPU/global_atomics.ll | 62 +-
.../test/CodeGen/AMDGPU/global_atomics_i64.ll | 36 +-
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 40 ++
.../test/CodeGen/AMDGPU/load-atomic-global.ll | 649 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/load-atomic-local.ll | 333 +++++++++
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 24 +
.../CodeGen/AMDGPU/store-atomic-global.ll | 398 +++++++++++
.../test/CodeGen/AMDGPU/store-atomic-local.ll | 250 +++++++
14 files changed, 1830 insertions(+), 173 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 434ad35f734995..069ebf72bc987b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -528,19 +528,11 @@ def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)> {
let IsLoad = 1;
}
-def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> {
- let IsAtomic = 1;
-}
-
-def atomic_load_nonext_32_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_32 node:$ptr)> {
- let IsAtomic = 1;
-}
-
-def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64 node:$ptr)> {
+def atomic_load_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> {
let IsAtomic = 1;
}
-def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> {
+def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> {
let IsAtomic = 1;
}
@@ -600,16 +592,14 @@ def truncstorei16_hi16_#as : StoreHi16<truncstorei16, i16>;
} // End let IsStore = 1, AddressSpaces = ...
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
+def atomic_store_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsTruncStore = 0;
+}
def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_8 node:$val, node:$ptr)>;
def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_16 node:$val, node:$ptr)>;
-def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_32 node:$val, node:$ptr)>;
-def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_64 node:$val, node:$ptr)>;
-def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_128 node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = ...
} // End foreach as
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 6cb0f77e0dbeb3..984adf838a55d8 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1084,10 +1084,12 @@ defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, atomic_load_sext_16_glo
foreach vt = Reg32Types.types in {
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>;
+defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, atomic_load_global>;
}
foreach vt = VReg_64.RegTypes in {
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>;
+defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, atomic_load_global>;
}
foreach vt = VReg_96.RegTypes in {
@@ -1133,10 +1135,12 @@ defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>;
foreach vt = Reg32Types.types in {
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>;
+defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, atomic_store_global>;
}
foreach vt = VReg_64.RegTypes in {
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>;
+defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, atomic_store_global>;
}
foreach vt = VReg_96.RegTypes in {
@@ -2061,19 +2065,6 @@ class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
(Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
>;
-multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
- ValueType vt, PatFrag atomic_ld> {
- def : GCNPat <
- (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))),
- (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
- >;
-
- def : GCNPat <
- (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))),
- (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset))
- >;
-}
-
let SubtargetPredicate = isGFX6GFX7 in {
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
@@ -2081,9 +2072,6 @@ def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constan
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
-
-defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_nonext_32_global>;
-defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_nonext_64_global>;
} // End SubtargetPredicate = isGFX6GFX7
multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt,
@@ -2255,8 +2243,6 @@ defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OF
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>;
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>;
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>;
} // End Predicates = isGFX6GFX7
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 6e6d9597e36c99..3ff2ef169ee8f0 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -1005,6 +1005,11 @@ defm : DSReadPat_t16 <DS_READ_U16, i16, "load_local">;
foreach vt = Reg32Types.types in {
defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
+defm : DSReadPat_mc <DS_READ_B32, vt, "atomic_load_local">;
+}
+
+foreach vt = VReg_64.RegTypes in {
+defm : DSReadPat_mc <DS_READ_B64, vt, "atomic_load_local">;
}
defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_aext_8_local">;
@@ -1017,11 +1022,6 @@ defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_nonext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;
defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;
-defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_nonext_32_local">;
-defm : DSReadPat_mc <DS_READ_B32, v2i16, "atomic_load_nonext_32_local">;
-defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_nonext_64_local">;
-defm : DSReadPat_mc <DS_READ_B64, v2i32, "atomic_load_nonext_64_local">;
-defm : DSReadPat_mc <DS_READ_B64, v4i16, "atomic_load_nonext_64_local">;
let OtherPredicates = [D16PreservesUnusedBits] in {
// TODO: Atomic loads
@@ -1098,17 +1098,17 @@ defm : DSWritePat_t16 <DS_WRITE_B16, i16, "store_local">;
foreach vt = Reg32Types.types in {
defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
+defm : DSWritePat_mc <DS_WRITE_B32, vt, "atomic_store_local">;
+}
+
+foreach vt = VReg_64.RegTypes in {
+defm : DSWritePat_mc <DS_WRITE_B64, vt, "atomic_store_local">;
}
defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">;
defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;
defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
-defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
-defm : DSWritePat_mc <DS_WRITE_B32, v2i16, "atomic_store_32_local">;
-defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
-defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "atomic_store_64_local">;
-defm : DSWritePat_mc <DS_WRITE_B64, v4i16, "atomic_store_64_local">;
let OtherPredicates = [HasD16LoadStore] in {
def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index f241f6677d22b2..5a431a27df8a97 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2107,24 +2107,21 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
}
-defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
-
defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
foreach vt = Reg32Types.types in {
defm : FlatLoadPats <FLAT_LOAD_DWORD, load_flat, vt>;
defm : FlatStorePats <FLAT_STORE_DWORD, store_flat, vt>;
+defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_flat, vt>;
+defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_flat, vt>;
}
foreach vt = VReg_64.RegTypes in {
defm : FlatStorePats <FLAT_STORE_DWORDX2, store_flat, vt>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, load_flat, vt>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_flat, vt>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>;
@@ -2132,14 +2129,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>;
foreach vt = VReg_128.RegTypes in {
defm : FlatLoadPats <FLAT_LOAD_DWORDX4, load_flat, vt>;
defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_flat, vt>;
+defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_flat, vt>;
}
-defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
-defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2271,14 +2264,21 @@ defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i
defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;
} // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts
+// There is no distinction for atomic load lowering during selection;
+// the memory legalizer will set the cache bits and insert the
+// appropriate waits.
foreach vt = Reg32Types.types in {
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_global, vt>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_global, vt>;
}
foreach vt = VReg_64.RegTypes in {
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_global, vt>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global, vt>;
}
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>;
@@ -2286,18 +2286,10 @@ defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>;
foreach vt = VReg_128.RegTypes in {
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_global, vt>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_global, vt>;
}
-// There is no distinction for atomic load lowering during selection;
-// the memory legalizer will set the cache bits and insert the
-// appropriate waits.
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
-
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>;
@@ -2340,12 +2332,6 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index e7425708d51a00..240d03478dc752 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -465,18 +465,6 @@ def atomic_load_nonext_16_glue : PatFrag<(ops node:$ptr),
let MemoryVT = i16;
}
-def atomic_load_nonext_32_glue : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_glue node:$ptr)> {
- let IsAtomic = 1;
- let MemoryVT = i32;
-}
-
-def atomic_load_nonext_64_glue : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_glue node:$ptr)> {
- let IsAtomic = 1;
- let MemoryVT = i64;
-}
-
def atomic_load_zext_8_glue : PatFrag<(ops node:$ptr),
(atomic_load_zext_glue node:$ptr)> {
let IsAtomic = 1;
@@ -587,12 +575,10 @@ def load_align16_#as#_m0 : PatFrag<(ops node:$ptr),
}
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
+def atomic_load_#as#_m0 : PatFrag<(ops node:$ptr),
+ (atomic_load_nonext_glue node:$ptr)>;
def atomic_load_nonext_16_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_nonext_16_glue node:$ptr)>;
-def atomic_load_nonext_32_#as#_m0 : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_32_glue node:$ptr)>;
-def atomic_load_nonext_64_#as#_m0 : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_64_glue node:$ptr)>;
def atomic_load_zext_8_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_zext_8_glue node:$ptr)>;
@@ -705,44 +691,35 @@ def store_align_less_than_4_#as#_m0 : PatFrag <(ops node:$value, node:$ptr),
}
}
-def atomic_store_8_glue : PatFrag <
- (ops node:$ptr, node:$value),
- (AMDGPUatomic_st_glue node:$ptr, node:$value)> {
- let IsAtomic = 1;
- let MemoryVT = i8;
-}
-
-def atomic_store_16_glue : PatFrag <
+def atomic_store_glue : PatFrag <
(ops node:$ptr, node:$value),
(AMDGPUatomic_st_glue node:$ptr, node:$value)> {
let IsAtomic = 1;
- let MemoryVT = i16;
+ let IsTruncStore = 0;
}
-def atomic_store_32_glue : PatFrag <
+def atomic_store_8_glue : PatFrag <
(ops node:$ptr, node:$value),
(AMDGPUatomic_st_glue node:$ptr, node:$value)> {
let IsAtomic = 1;
- let MemoryVT = i32;
+ let MemoryVT = i8;
}
-def atomic_store_64_glue : PatFrag <
+def atomic_store_16_glue : PatFrag <
(ops node:$ptr, node:$value),
(AMDGPUatomic_st_glue node:$ptr, node:$value)> {
let IsAtomic = 1;
- let MemoryVT = i64;
+ let MemoryVT = i16;
}
foreach as = [ "local", "region" ] in
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
+def atomic_store_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_glue node:$val, node:$ptr)>;
def atomic_store_8_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_8_glue node:$val, node:$ptr)>;
def atomic_store_16_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_16_glue node:$val, node:$ptr)>;
-def atomic_store_32_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_32_glue node:$val, node:$ptr)>;
-def atomic_store_64_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_64_glue node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = StoreAddress_#as.AddrSpaces
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
index 1dee03e4627b48..984184a9dd54d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
@@ -82,16 +82,26 @@ body: |
; GFX6-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX6: liveins: $vgpr0_vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX6-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX6-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
;
; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+ ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX7-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
@@ -255,16 +265,26 @@ body: |
; GFX6-LABEL: name: load_atomic_global_v2s32_seq_cst
; GFX6: liveins: $vgpr0_vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s32>), addrspace 1)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 1)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
;
; GFX7-LABEL: name: load_atomic_global_v2s32_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>), addrspace 1)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+ ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v2s32_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
@@ -306,16 +326,26 @@ body: |
; GFX6-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX6: liveins: $vgpr0_vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
;
; GFX7-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+ ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
;
; GFX7-FLAT-LABEL: name: load_atomic_global_v4s16_seq_cst
; GFX7-FLAT: liveins: $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index fdc8eff962d247..177c3bea9bf8a0 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -6528,15 +6528,13 @@ define amdgpu_kernel void @atomic_load_i32_offset(ptr addrspace(1) %in, ptr addr
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_load_dword v0, v[0:1] glc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_mov_b32 s4, s2
; VI-NEXT: s_mov_b32 s5, s3
+; VI-NEXT: s_mov_b32 s2, s6
+; VI-NEXT: s_mov_b32 s3, s7
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:16 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -6658,15 +6656,13 @@ define amdgpu_kernel void @atomic_load_f32_offset(ptr addrspace(1) %in, ptr addr
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_load_dword v0, v[0:1] glc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_mov_b32 s4, s2
; VI-NEXT: s_mov_b32 s5, s3
+; VI-NEXT: s_mov_b32 s2, s6
+; VI-NEXT: s_mov_b32 s3, s7
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:16 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -6722,9 +6718,9 @@ define amdgpu_kernel void @atomic_load_i32(ptr addrspace(1) %in, ptr addrspace(1
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_load_dword v0, v[0:1] glc
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: buffer_load_dword v0, off, s[4:7], 0 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_mov_b32 s4, s2
@@ -7020,15 +7016,13 @@ define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, ptr addrspace(1) %ou
;
; VI-LABEL: atomic_store_i32_offset:
; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dword s6, s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; VI-NEXT: s_load_dword s2, s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 offset:16
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_store_i32_offset:
@@ -7070,13 +7064,13 @@ define amdgpu_kernel void @atomic_store_i32(i32 %in, ptr addrspace(1) %out) {
;
; VI-LABEL: atomic_store_i32:
; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dword s6, s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; VI-NEXT: s_load_dword s2, s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_store_i32:
@@ -7117,13 +7111,13 @@ define amdgpu_kernel void @atomic_store_f32(float %in, ptr addrspace(1) %out) {
;
; VI-LABEL: atomic_store_f32:
; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dword s6, s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; VI-NEXT: s_load_dword s2, s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_store_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
index 0ad48c2d6b683a..600c69d10d5632 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
@@ -6380,15 +6380,13 @@ define amdgpu_kernel void @atomic_load_i64_offset(ptr addrspace(1) %in, ptr addr
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s0, s0, 32
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_mov_b32 s4, s2
; VI-NEXT: s_mov_b32 s5, s3
+; VI-NEXT: s_mov_b32 s2, s6
+; VI-NEXT: s_mov_b32 s3, s7
+; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[0:3], 0 offset:32 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -6508,9 +6506,9 @@ define amdgpu_kernel void @atomic_load_i64(ptr addrspace(1) %in, ptr addrspace(1
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_mov_b32 s4, s2
@@ -6802,14 +6800,14 @@ define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, ptr addrspace(1) %ou
; VI-LABEL: atomic_store_i64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_add_u32 s0, s2, 32
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_addc_u32 s1, s3, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_mov_b32 s4, s2
+; VI-NEXT: s_mov_b32 s5, s3
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 offset:32
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_store_i64_offset:
@@ -6855,12 +6853,14 @@ define amdgpu_kernel void @atomic_store_i64(i64 %in, ptr addrspace(1) %out) {
; VI-LABEL: atomic_store_i64:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_mov_b32 s4, s2
+; VI-NEXT: s_mov_b32 s5, s3
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_store_i64:
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
index ce41cfc7b2c249..31377cf7349557 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -419,6 +419,46 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
store i16 %sum, ptr addrspace(1) %out, align 4
ret void
}
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x i32>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x i32> %a0, i32 0
+ %num2 = extractelement <2 x i32> %a0, i32 1
+ %res = add i32 %num1, %num2
+ store i32 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index 6d0f7cb4a6c4e3..76f5555352c7c6 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -1,4 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
@@ -9,6 +11,29 @@
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
@@ -49,6 +74,31 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, pt
}
define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_wbinvl1
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: buffer_wbinvl1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
@@ -95,6 +145,29 @@ define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr
}
define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
@@ -135,6 +208,37 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p
}
define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
@@ -175,6 +279,39 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, pt
}
define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_wbinvl1
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: buffer_wbinvl1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
@@ -221,6 +358,37 @@ define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(1) %p, ptr
}
define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off
@@ -261,6 +429,31 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(1) %p
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
@@ -302,6 +495,33 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(1) %p, pt
}
define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_wbinvl1
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: buffer_wbinvl1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off glc
@@ -349,6 +569,31 @@ define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(1) %p, ptr
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off
@@ -390,6 +635,55 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(1) %p
}
define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-SDAG-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-GISEL-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
@@ -444,6 +738,57 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(1) %p, pt
}
define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_wbinvl1
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-SDAG-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: buffer_wbinvl1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-GISEL-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
@@ -504,6 +849,55 @@ define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(1) %p, ptr
}
define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-SDAG-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-GISEL-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
@@ -558,6 +952,37 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(1) %p
}
define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-SDAG-NEXT: v_mad_u32_u24 v0, v1, v5, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_mad_u32_u24 v0, v1, v5, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
@@ -611,6 +1036,39 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(1) %p, pt
}
define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_wbinvl1
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-SDAG-NEXT: v_mad_u32_u24 v0, v1, v5, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: buffer_wbinvl1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_mad_u32_u24 v0, v1, v5, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
@@ -670,6 +1128,37 @@ define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(1) %p, ptr
}
define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-SDAG-NEXT: v_mad_u32_u24 v0, v1, v5, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_mad_u32_u24 v0, v1, v5, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
@@ -723,6 +1212,29 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(1) %p
}
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:1 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:1 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc
@@ -764,6 +1276,29 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(
}
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:4095 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:4095 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:4095 glc
@@ -817,6 +1352,31 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspac
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:1 glc
@@ -859,6 +1419,31 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:4095 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:4095 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4095 glc
@@ -912,11 +1497,75 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
store i16 %sum, ptr addrspace(1) %out, align 4
ret void
}
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x i32>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x i32> %a0, i32 0
+ %num2 = extractelement <2 x i32> %a0, i32 1
+ %res = add i32 %num1, %num2
+ store i32 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
+; GFX6: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
index 05bb7ff9de4d3a..caaf919daef49b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
@@ -1,4 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
@@ -9,6 +11,15 @@
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f32x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -49,6 +60,15 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(3) %p, pt
}
define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f32x2_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -92,6 +112,15 @@ define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(3) %p, ptr
}
define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f32x2_monotonic_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -132,6 +161,32 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(3) %p
}
define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_read_b32 v0, v0
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: ds_write_b16 v1, v0
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_read_b32 v0, v0
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: ds_write_b16 v1, v0
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0
@@ -172,6 +227,32 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(3) %p, pt
}
define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_read_b32 v0, v0
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: ds_write_b16 v1, v0
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_read_b32 v0, v0
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: ds_write_b16 v1, v0
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0
@@ -215,6 +296,32 @@ define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(3) %p, ptr
}
define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_read_b32 v0, v0
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: ds_write_b16 v1, v0
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_read_b32 v0, v0
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-GISEL-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: ds_write_b16 v1, v0
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0
@@ -255,6 +362,16 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(3) %p
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0
@@ -296,6 +413,16 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(3) %p, pt
}
define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0
@@ -340,6 +467,16 @@ define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(3) %p, ptr
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0
@@ -381,6 +518,28 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(3) %p
}
define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v4
+; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -435,6 +594,28 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(3) %p, pt
}
define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v4
+; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -492,6 +673,28 @@ define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(3) %p, ptr
}
define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v4
+; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -546,6 +749,19 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(3) %p
}
define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX6-NEXT: v_mad_u32_u24 v0, v2, v4, v0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -599,6 +815,19 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(3) %p, pt
}
define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX6-NEXT: v_mad_u32_u24 v0, v2, v4, v0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -655,6 +884,19 @@ define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(3) %p, ptr
}
define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX6-NEXT: v_mad_u32_u24 v0, v2, v4, v0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0
@@ -708,6 +950,16 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(3) %p
}
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0 offset:1
@@ -749,6 +1001,16 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(
}
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfff, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_add_f32_e32 v0, v2, v3
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b64 v[2:3], v0 offset:4095
@@ -790,6 +1052,17 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspac
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0 offset:1
@@ -832,6 +1105,17 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(
}
define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfff, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_read_b32 v0, v0 offset:4095
@@ -872,6 +1156,55 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
store i16 %sum, ptr addrspace(3) %out, align 4
ret void
}
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v3
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read_b64 v[2:3], v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v0, v2, v3
+; GFX9-NEXT: ds_write_b32 v1, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_read_b64 v[2:3], v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v2, v3
+; GFX10-NEXT: ds_write_b32 v1, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v2, v3
+; GFX11-NEXT: ds_store_b32 v1, v0
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, v2, v3
+; GFX12-NEXT: ds_store_b32 v1, v0
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x i32>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x i32> %a0, i32 0
+ %num2 = extractelement <2 x i32> %a0, i32 1
+ %res = add i32 %num1, %num2
+ store i32 %res, ptr addrspace(3) %out, align 4
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
index 855091bb8a0053..3e69279b03a2fa 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -186,6 +186,30 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %
store atomic <4 x i16> %in, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
ret void
}
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i32> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
index 3de24b553f793a..9249e94a5ea3bb 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -1,4 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
@@ -9,6 +11,23 @@
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -33,6 +52,23 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
}
define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -57,6 +93,23 @@ define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr add
}
define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -81,6 +134,23 @@ define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr
}
define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dword v[1:2], v0, off
@@ -105,6 +175,23 @@ define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr ad
}
define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dword v[1:2], v0, off
@@ -129,6 +216,23 @@ define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addr
}
define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dword v[1:2], v0, off
@@ -153,6 +257,23 @@ define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, pt
}
define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dword v[1:2], v0, off
@@ -177,6 +298,23 @@ define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr add
}
define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dword v[1:2], v0, off
@@ -201,6 +339,23 @@ define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrs
}
define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dword v[1:2], v0, off
@@ -225,6 +380,31 @@ define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr
}
define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -249,6 +429,31 @@ define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr ad
}
define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -273,6 +478,31 @@ define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addr
}
define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -297,6 +527,31 @@ define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, pt
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -321,6 +576,31 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr add
}
define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -345,6 +625,31 @@ define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrs
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
@@ -369,6 +674,32 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_movk_i32 s0, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s1, -1
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_movk_i32 s0, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b32 s1, -1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:-4096
@@ -396,6 +727,31 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:4088
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:4088
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:4088
@@ -428,11 +784,53 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %
store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
ret void
}
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(1) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s0, s2
+; GFX6-SDAG-NEXT: s_mov_b32 s1, s2
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
+; GFX6-GISEL-NEXT: s_endpgm
+;
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i32> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
+; GFX6: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
index eb04e3980ad2cd..7712ea80bf6b71 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-local.ll
@@ -1,4 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
@@ -9,6 +11,12 @@
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_f32x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -33,6 +41,12 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
}
define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_f32x2_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -57,6 +71,12 @@ define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr add
}
define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_f32x2_seq_cst_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -81,6 +101,12 @@ define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr
}
define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_f16x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b32 v1, v0
@@ -105,6 +131,12 @@ define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr ad
}
define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_f16x2_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b32 v1, v0
@@ -129,6 +161,12 @@ define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addr
}
define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_f16x2_monotonic_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b32 v1, v0
@@ -153,6 +191,12 @@ define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, pt
}
define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_i16x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b32 v1, v0
@@ -177,6 +221,12 @@ define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr add
}
define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_i16x2_seq_cst_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b32 v1, v0
@@ -201,6 +251,12 @@ define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrs
}
define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_i16x2_monotonic_wavefront:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b32 v1, v0
@@ -225,6 +281,26 @@ define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr
}
define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x4_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -249,6 +325,26 @@ define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr ad
}
define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x4_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -273,6 +369,26 @@ define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addr
}
define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_f16x4_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -297,6 +413,26 @@ define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, pt
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -321,6 +457,26 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr add
}
define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_seq_cst_agent:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -345,6 +501,26 @@ define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrs
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_wavefront:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -369,6 +545,28 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_1(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_1:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:1
@@ -394,6 +592,28 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_1(<4 x i16> %in
}
define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(3) %out) {
+; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xff8, v2
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX6-SDAG-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xff8, v2
+; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX6-GISEL-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-GISEL-NEXT: s_endpgm
+;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX9: ; %bb.0:
; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:4088
@@ -417,6 +637,36 @@ define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %
store atomic <4 x i16> %in, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
ret void
}
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(3) %out) {
+; GFX6-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: ds_store_b64 v2, v[0:1]
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i32> %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
More information about the llvm-branch-commits
mailing list