[llvm-branch-commits] [llvm] AMDGPU: Use normal load/store pattern type lists for atomics (PR #229688)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 7 01:20:35 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Matt Arsenault (arsenm)
<details>
<summary>Changes</summary>
Atomic load and store of vector types are now permitted in the IR. Instead of
maintaining separate scalar-only atomic pattern lists, cover atomic load/store
in the existing per-register-type pattern loops. As a side effect
-flat-for-global is respected in more cases.
Co-authored-by: Claude Opus 5.5 <noreply@<!-- -->anthropic.com>
---
Patch is 130.09 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/229688.diff
14 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUInstructions.td (+6-16)
- (modified) llvm/lib/Target/AMDGPU/BUFInstructions.td (+4-18)
- (modified) llvm/lib/Target/AMDGPU/DSInstructions.td (+10-10)
- (modified) llvm/lib/Target/AMDGPU/FLATInstructions.td (+15-29)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.td (+10-33)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir (+45-15)
- (modified) llvm/test/CodeGen/AMDGPU/global_atomics.ll (+28-34)
- (modified) llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll (+18-18)
- (modified) llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll (+40)
- (modified) llvm/test/CodeGen/AMDGPU/load-atomic-global.ll (+649)
- (modified) llvm/test/CodeGen/AMDGPU/load-atomic-local.ll (+333)
- (modified) llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll (+24)
- (modified) llvm/test/CodeGen/AMDGPU/store-atomic-global.ll (+398)
- (modified) llvm/test/CodeGen/AMDGPU/store-atomic-local.ll (+250)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 434ad35f73499..069ebf72bc987 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -528,19 +528,11 @@ def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)> {
let IsLoad = 1;
}
-def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> {
- let IsAtomic = 1;
-}
-
-def atomic_load_nonext_32_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_32 node:$ptr)> {
- let IsAtomic = 1;
-}
-
-def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64 node:$ptr)> {
+def atomic_load_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> {
let IsAtomic = 1;
}
-def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> {
+def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> {
let IsAtomic = 1;
}
@@ -600,16 +592,14 @@ def truncstorei16_hi16_#as : StoreHi16<truncstorei16, i16>;
} // End let IsStore = 1, AddressSpaces = ...
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
+def atomic_store_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsTruncStore = 0;
+}
def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_8 node:$val, node:$ptr)>;
def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_16 node:$val, node:$ptr)>;
-def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_32 node:$val, node:$ptr)>;
-def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_64 node:$val, node:$ptr)>;
-def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_128 node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = ...
} // End foreach as
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 6cb0f77e0dbeb..984adf838a55d 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1084,10 +1084,12 @@ defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, atomic_load_sext_16_glo
foreach vt = Reg32Types.types in {
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>;
+defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, atomic_load_global>;
}
foreach vt = VReg_64.RegTypes in {
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>;
+defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, atomic_load_global>;
}
foreach vt = VReg_96.RegTypes in {
@@ -1133,10 +1135,12 @@ defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>;
foreach vt = Reg32Types.types in {
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>;
+defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, atomic_store_global>;
}
foreach vt = VReg_64.RegTypes in {
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>;
+defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, atomic_store_global>;
}
foreach vt = VReg_96.RegTypes in {
@@ -2061,19 +2065,6 @@ class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
(Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
>;
-multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
- ValueType vt, PatFrag atomic_ld> {
- def : GCNPat <
- (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))),
- (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
- >;
-
- def : GCNPat <
- (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))),
- (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset))
- >;
-}
-
let SubtargetPredicate = isGFX6GFX7 in {
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
@@ -2081,9 +2072,6 @@ def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constan
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
-
-defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_nonext_32_global>;
-defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_nonext_64_global>;
} // End SubtargetPredicate = isGFX6GFX7
multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt,
@@ -2255,8 +2243,6 @@ defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OF
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>;
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>;
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>;
} // End Predicates = isGFX6GFX7
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 6e6d9597e36c9..3ff2ef169ee8f 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -1005,6 +1005,11 @@ defm : DSReadPat_t16 <DS_READ_U16, i16, "load_local">;
foreach vt = Reg32Types.types in {
defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
+defm : DSReadPat_mc <DS_READ_B32, vt, "atomic_load_local">;
+}
+
+foreach vt = VReg_64.RegTypes in {
+defm : DSReadPat_mc <DS_READ_B64, vt, "atomic_load_local">;
}
defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_aext_8_local">;
@@ -1017,11 +1022,6 @@ defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_nonext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;
defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;
-defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_nonext_32_local">;
-defm : DSReadPat_mc <DS_READ_B32, v2i16, "atomic_load_nonext_32_local">;
-defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_nonext_64_local">;
-defm : DSReadPat_mc <DS_READ_B64, v2i32, "atomic_load_nonext_64_local">;
-defm : DSReadPat_mc <DS_READ_B64, v4i16, "atomic_load_nonext_64_local">;
let OtherPredicates = [D16PreservesUnusedBits] in {
// TODO: Atomic loads
@@ -1098,17 +1098,17 @@ defm : DSWritePat_t16 <DS_WRITE_B16, i16, "store_local">;
foreach vt = Reg32Types.types in {
defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
+defm : DSWritePat_mc <DS_WRITE_B32, vt, "atomic_store_local">;
+}
+
+foreach vt = VReg_64.RegTypes in {
+defm : DSWritePat_mc <DS_WRITE_B64, vt, "atomic_store_local">;
}
defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">;
defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;
defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
-defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
-defm : DSWritePat_mc <DS_WRITE_B32, v2i16, "atomic_store_32_local">;
-defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
-defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "atomic_store_64_local">;
-defm : DSWritePat_mc <DS_WRITE_B64, v4i16, "atomic_store_64_local">;
let OtherPredicates = [HasD16LoadStore] in {
def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index f241f6677d22b..5a431a27df8a9 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2107,24 +2107,21 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
}
-defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
-
defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
foreach vt = Reg32Types.types in {
defm : FlatLoadPats <FLAT_LOAD_DWORD, load_flat, vt>;
defm : FlatStorePats <FLAT_STORE_DWORD, store_flat, vt>;
+defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_flat, vt>;
+defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_flat, vt>;
}
foreach vt = VReg_64.RegTypes in {
defm : FlatStorePats <FLAT_STORE_DWORDX2, store_flat, vt>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, load_flat, vt>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_flat, vt>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>;
@@ -2132,14 +2129,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>;
foreach vt = VReg_128.RegTypes in {
defm : FlatLoadPats <FLAT_LOAD_DWORDX4, load_flat, vt>;
defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_flat, vt>;
+defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_flat, vt>;
}
-defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
-defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2271,14 +2264,21 @@ defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i
defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;
} // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts
+// There is no distinction for atomic load lowering during selection;
+// the memory legalizer will set the cache bits and insert the
+// appropriate waits.
foreach vt = Reg32Types.types in {
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_global, vt>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_global, vt>;
}
foreach vt = VReg_64.RegTypes in {
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_global, vt>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global, vt>;
}
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>;
@@ -2286,18 +2286,10 @@ defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>;
foreach vt = VReg_128.RegTypes in {
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_global, vt>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_global, vt>;
}
-// There is no distinction for atomic load lowering during selection;
-// the memory legalizer will set the cache bits and insert the
-// appropriate waits.
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
-
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>;
@@ -2340,12 +2332,6 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index e7425708d51a0..240d03478dc75 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -465,18 +465,6 @@ def atomic_load_nonext_16_glue : PatFrag<(ops node:$ptr),
let MemoryVT = i16;
}
-def atomic_load_nonext_32_glue : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_glue node:$ptr)> {
- let IsAtomic = 1;
- let MemoryVT = i32;
-}
-
-def atomic_load_nonext_64_glue : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_glue node:$ptr)> {
- let IsAtomic = 1;
- let MemoryVT = i64;
-}
-
def atomic_load_zext_8_glue : PatFrag<(ops node:$ptr),
(atomic_load_zext_glue node:$ptr)> {
let IsAtomic = 1;
@@ -587,12 +575,10 @@ def load_align16_#as#_m0 : PatFrag<(ops node:$ptr),
}
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
+def atomic_load_#as#_m0 : PatFrag<(ops node:$ptr),
+ (atomic_load_nonext_glue node:$ptr)>;
def atomic_load_nonext_16_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_nonext_16_glue node:$ptr)>;
-def atomic_load_nonext_32_#as#_m0 : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_32_glue node:$ptr)>;
-def atomic_load_nonext_64_#as#_m0 : PatFrag<(ops node:$ptr),
- (atomic_load_nonext_64_glue node:$ptr)>;
def atomic_load_zext_8_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_zext_8_glue node:$ptr)>;
@@ -705,44 +691,35 @@ def store_align_less_than_4_#as#_m0 : PatFrag <(ops node:$value, node:$ptr),
}
}
-def atomic_store_8_glue : PatFrag <
- (ops node:$ptr, node:$value),
- (AMDGPUatomic_st_glue node:$ptr, node:$value)> {
- let IsAtomic = 1;
- let MemoryVT = i8;
-}
-
-def atomic_store_16_glue : PatFrag <
+def atomic_store_glue : PatFrag <
(ops node:$ptr, node:$value),
(AMDGPUatomic_st_glue node:$ptr, node:$value)> {
let IsAtomic = 1;
- let MemoryVT = i16;
+ let IsTruncStore = 0;
}
-def atomic_store_32_glue : PatFrag <
+def atomic_store_8_glue : PatFrag <
(ops node:$ptr, node:$value),
(AMDGPUatomic_st_glue node:$ptr, node:$value)> {
let IsAtomic = 1;
- let MemoryVT = i32;
+ let MemoryVT = i8;
}
-def atomic_store_64_glue : PatFrag <
+def atomic_store_16_glue : PatFrag <
(ops node:$ptr, node:$value),
(AMDGPUatomic_st_glue node:$ptr, node:$value)> {
let IsAtomic = 1;
- let MemoryVT = i64;
+ let MemoryVT = i16;
}
foreach as = [ "local", "region" ] in
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
+def atomic_store_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_glue node:$val, node:$ptr)>;
def atomic_store_8_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_8_glue node:$val, node:$ptr)>;
def atomic_store_16_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_16_glue node:$val, node:$ptr)>;
-def atomic_store_32_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_32_glue node:$val, node:$ptr)>;
-def atomic_store_64_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_64_glue node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = StoreAddress_#as.AddrSpaces
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
index 1dee03e4627b4..984184a9dd54d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir
@@ -82,16 +82,26 @@ body: |
; GFX6-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX6: liveins: $vgpr0_vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX6-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+ ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+ ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX6-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
;
; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst
; GFX7: liveins: $vgpr0_vgpr1
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
- ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1)
- ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+ ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+ ; GFX7-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1)
+ ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
;
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/229688
More information about the llvm-branch-commits
mailing list