[llvm] [AMDGPU] Use vector memory types for 64/128-bit cooperative atomics (PR #229501)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 10:49:26 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-selectiondag
Author: Gheorghe-Teodor Bercea (doru1004)
<details>
<summary>Changes</summary>
The 16x8B and 8x16B cooperative atomic intrinsics had integer memory types (`i64`/`i128`) that didn't match their vector values, which broke value tracking on the loaded value and could crash the compiler. This patch uses the value type as the memory type and updates the selection patterns to match. Codegen for existing cooperative atomics is unchanged.
---
Patch is 28.09 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/229501.diff
9 Files Affected:
- (modified) llvm/include/llvm/Target/TargetSelectionDAG.td (+16-8)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUInstructions.td (+9-3)
- (modified) llvm/lib/Target/AMDGPU/FLATInstructions.td (+8-8)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+4-8)
- (added) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll (+286)
- (modified) llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll (+40)
- (modified) llvm/test/CodeGen/AMDGPU/load-atomic-global.ll (+40)
- (modified) llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll (+24)
- (modified) llvm/test/CodeGen/AMDGPU/store-atomic-global.ll (+24)
``````````diff
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 4f8a4987ff838..352fbcf72a22b 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -2165,6 +2165,13 @@ def atomic_load_64 :
let MemoryVT = i64;
}
+def atomic_load_64_v2i32 :
+ PatFrag<(ops node:$ptr),
+ (atomic_load node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = v2i32;
+}
+
def atomic_load_128_v2i64 :
PatFrag<(ops node:$ptr),
(atomic_load node:$ptr)> {
@@ -2203,12 +2210,6 @@ def atomic_load_nonext_64 :
let MemoryVT = i64;
}
-def atomic_load_nonext_128 :
- PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> {
- let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
- let MemoryVT = i128;
-}
-
def atomic_load_zext_8 :
PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
@@ -2440,11 +2441,18 @@ def atomic_store_64 :
let MemoryVT = i64;
}
-def atomic_store_128 :
+def atomic_store_64_v2i32 :
PatFrag<(ops node:$val, node:$ptr),
(atomic_store node:$val, node:$ptr)> {
let IsAtomic = true;
- let MemoryVT = i128;
+ let MemoryVT = v2i32;
+}
+
+def atomic_store_128_v4i32 :
+ PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = v4i32;
}
// Matches store or atomic_store, no alignment requirement.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 434ad35f73499..6833020be0aa4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -540,7 +540,11 @@ def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64
let IsAtomic = 1;
}
-def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> {
+def atomic_load_64_v2i32_#as : PatFrag<(ops node:$ptr), (atomic_load_64_v2i32 node:$ptr)> {
+ let IsAtomic = 1;
+}
+
+def atomic_load_128_v4i32_#as : PatFrag<(ops node:$ptr), (atomic_load_128_v4i32 node:$ptr)> {
let IsAtomic = 1;
}
@@ -608,8 +612,10 @@ def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_32 node:$val, node:$ptr)>;
def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_64 node:$val, node:$ptr)>;
-def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_128 node:$val, node:$ptr)>;
+def atomic_store_64_v2i32_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_64_v2i32 node:$val, node:$ptr)>;
+def atomic_store_128_v4i32_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_128_v4i32 node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = ...
} // End foreach as
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index f241f6677d22b..fb3ea1c0055a3 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2111,8 +2111,8 @@ defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_64_v2i32_flat, v2i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_128_v4i32_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
@@ -2137,9 +2137,9 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_v2i32_flat, v2i32>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_v4i32_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2294,9 +2294,9 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_v2i32_global, v2i32>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_128_v4i32_global, v4i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
@@ -2344,8 +2344,8 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_v2i32_global, v2i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_v4i32_global, v4i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 31ab77d5e900b..64eb21d4c2675 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1443,24 +1443,18 @@ static unsigned getIntrMemWidth(unsigned IntrID) {
case Intrinsic::amdgcn_global_load_async_to_lds_b32:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
case Intrinsic::amdgcn_global_store_async_from_lds_b32:
- case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
- case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
case Intrinsic::amdgcn_flat_load_monitor_b32:
case Intrinsic::amdgcn_global_load_monitor_b32:
return 32;
case Intrinsic::amdgcn_global_load_async_to_lds_b64:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
case Intrinsic::amdgcn_global_store_async_from_lds_b64:
- case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
- case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
case Intrinsic::amdgcn_flat_load_monitor_b64:
case Intrinsic::amdgcn_global_load_monitor_b64:
return 64;
case Intrinsic::amdgcn_global_load_async_to_lds_b128:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
case Intrinsic::amdgcn_global_store_async_from_lds_b128:
- case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
- case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
case Intrinsic::amdgcn_flat_load_monitor_b128:
case Intrinsic::amdgcn_global_load_monitor_b128:
return 128;
@@ -1781,7 +1775,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
+ // These are lowered to generic atomic loads and stores, which require the
+ // memory type to match the value type.
+ Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0);
Info.align.reset();
Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
@@ -1794,7 +1790,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
Info.opc = ISD::INTRINSIC_VOID;
- Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
+ Info.memVT = MVT::getVT(CI.getArgOperand(1)->getType());
Info.ptrVal = CI.getArgOperand(0);
Info.align.reset();
Info.flags = (MachineMemOperand::MOStore | MOCooperative);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
new file mode 100644
index 0000000000000..cbadd490b89ba
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+; 64-bit and 128-bit cooperative atomics on global and flat pointers, including
+; uses of the loaded value that query known bits and sign bits, in the same
+; block and across blocks.
+
+define <2 x i32> @load_16x8B_global(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ ret <2 x i32> %val
+}
+
+define <4 x i32> @load_8x16B_global(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ ret <4 x i32> %val
+}
+
+define void @store_16x8B_global(ptr addrspace(1) %addr, <2 x i32> %val) {
+; GFX1250-LABEL: store_16x8B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.cooperative.atomic.store.16x8B.p1(ptr addrspace(1) %addr, <2 x i32> %val, i32 0, metadata !0)
+ ret void
+}
+
+define void @store_8x16B_global(ptr addrspace(1) %addr, <4 x i32> %val) {
+; GFX1250-LABEL: store_8x16B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.cooperative.atomic.store.8x16B.p1(ptr addrspace(1) %addr, <4 x i32> %val, i32 0, metadata !0)
+ ret void
+}
+
+define i32 @load_16x8B_global_extract_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @load_16x8B_flat_extract_and(ptr %addr) {
+; GFX1250-LABEL: load_16x8B_flat_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p0(ptr %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @load_8x16B_global_extract_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @load_8x16B_flat_extract_and(ptr %addr) {
+; GFX1250-LABEL: load_8x16B_flat_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: flat_load_b128 v[0:3], v[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p0(ptr %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i64 @load_8x16B_global_bitcast_v2i64_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_bitcast_v2i64_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_mov_b32 s0, 0xff
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %cast = bitcast <4 x i32> %val to <2 x i64>
+ %elt = extractelement <2 x i64> %cast, i64 0
+ %and = and i64 %elt, 255
+ ret i64 %and
+}
+
+define i32 @load_16x8B_global_sext_inreg(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global_sext_inreg:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v0, v1, 0, 16
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %val, i32 1
+ %shl = shl i32 %elt, 16
+ %ashr = ashr i32 %shl, 16
+ ret i32 %ashr
+}
+
+define i32 @load_8x16B_global_sext_inreg(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_sext_inreg:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v0, v3, 0, 16
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %val, i32 3
+ %shl = shl i32 %elt, 16
+ %ashr = ashr i32 %shl, 16
+ ret i32 %ashr
+}
+
+define void @load_16x8B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: load_16x8B_global_live_out:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v4
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB11_2
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT: .LBB11_2: ; %exit
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: load_16x8B_global_live_out:
+; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v4
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB11_2
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %use
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-GISEL-NEXT: .LBB11_2: ; %exit
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ br i1 %cond, label %use, label %exit
+
+use:
+ store <2 x i32> %val, ptr addrspace(1) %out
+ br label %exit
+
+exit:
+ ret void
+}
+
+define void @load_8x16B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: load_8x16B_global_live_out:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB12_2
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1250-SDAG-NEXT: .LBB12_2: ; %exit
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: load_8x16B...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/229501
More information about the llvm-commits
mailing list