[llvm] [AMDGPU] Use vector memory types for 64/128-bit cooperative atomics (PR #229501)
Gheorghe-Teodor Bercea via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 11:54:01 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/229501
>From cc6c707183874c32446d67bc186b45b9e0c924f0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 6 Oct 2026 13:11:51 -0400
Subject: [PATCH 1/2] Use vector memory types for 64/128-bit cooperative
atomics
---
.../include/llvm/Target/TargetSelectionDAG.td | 24 +-
llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 12 +-
llvm/lib/Target/AMDGPU/FLATInstructions.td | 16 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +-
.../llvm.amdgcn.cooperative.atomic-vector.ll | 286 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 40 +++
.../test/CodeGen/AMDGPU/load-atomic-global.ll | 40 +++
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 24 ++
.../CodeGen/AMDGPU/store-atomic-global.ll | 24 ++
9 files changed, 451 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 4f8a4987ff838a5..352fbcf72a22ba1 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -2165,6 +2165,13 @@ def atomic_load_64 :
let MemoryVT = i64;
}
+def atomic_load_64_v2i32 :
+ PatFrag<(ops node:$ptr),
+ (atomic_load node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = v2i32;
+}
+
def atomic_load_128_v2i64 :
PatFrag<(ops node:$ptr),
(atomic_load node:$ptr)> {
@@ -2203,12 +2210,6 @@ def atomic_load_nonext_64 :
let MemoryVT = i64;
}
-def atomic_load_nonext_128 :
- PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> {
- let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
- let MemoryVT = i128;
-}
-
def atomic_load_zext_8 :
PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
@@ -2440,11 +2441,18 @@ def atomic_store_64 :
let MemoryVT = i64;
}
-def atomic_store_128 :
+def atomic_store_64_v2i32 :
PatFrag<(ops node:$val, node:$ptr),
(atomic_store node:$val, node:$ptr)> {
let IsAtomic = true;
- let MemoryVT = i128;
+ let MemoryVT = v2i32;
+}
+
+def atomic_store_128_v4i32 :
+ PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = v4i32;
}
// Matches store or atomic_store, no alignment requirement.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 434ad35f734995c..6833020be0aa415 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -540,7 +540,11 @@ def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64
let IsAtomic = 1;
}
-def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> {
+def atomic_load_64_v2i32_#as : PatFrag<(ops node:$ptr), (atomic_load_64_v2i32 node:$ptr)> {
+ let IsAtomic = 1;
+}
+
+def atomic_load_128_v4i32_#as : PatFrag<(ops node:$ptr), (atomic_load_128_v4i32 node:$ptr)> {
let IsAtomic = 1;
}
@@ -608,8 +612,10 @@ def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_32 node:$val, node:$ptr)>;
def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
(atomic_store_64 node:$val, node:$ptr)>;
-def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_128 node:$val, node:$ptr)>;
+def atomic_store_64_v2i32_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_64_v2i32 node:$val, node:$ptr)>;
+def atomic_store_128_v4i32_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_128_v4i32 node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = ...
} // End foreach as
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index f241f6677d22b2a..fb3ea1c0055a330 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2111,8 +2111,8 @@ defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_64_v2i32_flat, v2i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_128_v4i32_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
@@ -2137,9 +2137,9 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_v2i32_flat, v2i32>;
defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_v4i32_flat, v4i32>;
defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
@@ -2294,9 +2294,9 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_v2i32_global, v2i32>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_128_v4i32_global, v4i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
@@ -2344,8 +2344,8 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_v2i32_global, v2i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_v4i32_global, v4i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 31ab77d5e900bb0..64eb21d4c2675eb 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1443,24 +1443,18 @@ static unsigned getIntrMemWidth(unsigned IntrID) {
case Intrinsic::amdgcn_global_load_async_to_lds_b32:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
case Intrinsic::amdgcn_global_store_async_from_lds_b32:
- case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
- case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
case Intrinsic::amdgcn_flat_load_monitor_b32:
case Intrinsic::amdgcn_global_load_monitor_b32:
return 32;
case Intrinsic::amdgcn_global_load_async_to_lds_b64:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
case Intrinsic::amdgcn_global_store_async_from_lds_b64:
- case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
- case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
case Intrinsic::amdgcn_flat_load_monitor_b64:
case Intrinsic::amdgcn_global_load_monitor_b64:
return 64;
case Intrinsic::amdgcn_global_load_async_to_lds_b128:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
case Intrinsic::amdgcn_global_store_async_from_lds_b128:
- case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
- case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
case Intrinsic::amdgcn_flat_load_monitor_b128:
case Intrinsic::amdgcn_global_load_monitor_b128:
return 128;
@@ -1781,7 +1775,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
+ // These are lowered to generic atomic loads and stores, which require the
+ // memory type to match the value type.
+ Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0);
Info.align.reset();
Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
@@ -1794,7 +1790,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
Info.opc = ISD::INTRINSIC_VOID;
- Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
+ Info.memVT = MVT::getVT(CI.getArgOperand(1)->getType());
Info.ptrVal = CI.getArgOperand(0);
Info.align.reset();
Info.flags = (MachineMemOperand::MOStore | MOCooperative);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
new file mode 100644
index 000000000000000..cbadd490b89ba59
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+; 64-bit and 128-bit cooperative atomics on global and flat pointers, including
+; uses of the loaded value that query known bits and sign bits, in the same
+; block and across blocks.
+
+define <2 x i32> @load_16x8B_global(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ ret <2 x i32> %val
+}
+
+define <4 x i32> @load_8x16B_global(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ ret <4 x i32> %val
+}
+
+define void @store_16x8B_global(ptr addrspace(1) %addr, <2 x i32> %val) {
+; GFX1250-LABEL: store_16x8B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.cooperative.atomic.store.16x8B.p1(ptr addrspace(1) %addr, <2 x i32> %val, i32 0, metadata !0)
+ ret void
+}
+
+define void @store_8x16B_global(ptr addrspace(1) %addr, <4 x i32> %val) {
+; GFX1250-LABEL: store_8x16B_global:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.cooperative.atomic.store.8x16B.p1(ptr addrspace(1) %addr, <4 x i32> %val, i32 0, metadata !0)
+ ret void
+}
+
+define i32 @load_16x8B_global_extract_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @load_16x8B_flat_extract_and(ptr %addr) {
+; GFX1250-LABEL: load_16x8B_flat_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p0(ptr %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @load_8x16B_global_extract_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @load_8x16B_flat_extract_and(ptr %addr) {
+; GFX1250-LABEL: load_8x16B_flat_extract_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: flat_load_b128 v[0:3], v[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p0(ptr %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %val, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i64 @load_8x16B_global_bitcast_v2i64_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_bitcast_v2i64_and:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_mov_b32 s0, 0xff
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %cast = bitcast <4 x i32> %val to <2 x i64>
+ %elt = extractelement <2 x i64> %cast, i64 0
+ %and = and i64 %elt, 255
+ ret i64 %and
+}
+
+define i32 @load_16x8B_global_sext_inreg(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global_sext_inreg:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v0, v1, 0, 16
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %val, i32 1
+ %shl = shl i32 %elt, 16
+ %ashr = ashr i32 %shl, 16
+ ret i32 %ashr
+}
+
+define i32 @load_8x16B_global_sext_inreg(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_sext_inreg:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v0, v3, 0, 16
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %val, i32 3
+ %shl = shl i32 %elt, 16
+ %ashr = ashr i32 %shl, 16
+ ret i32 %ashr
+}
+
+define void @load_16x8B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: load_16x8B_global_live_out:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v4
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB11_2
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT: .LBB11_2: ; %exit
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: load_16x8B_global_live_out:
+; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v4
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB11_2
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %use
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-GISEL-NEXT: .LBB11_2: ; %exit
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ br i1 %cond, label %use, label %exit
+
+use:
+ store <2 x i32> %val, ptr addrspace(1) %out
+ br label %exit
+
+exit:
+ ret void
+}
+
+define void @load_8x16B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: load_8x16B_global_live_out:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB12_2
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1250-SDAG-NEXT: .LBB12_2: ; %exit
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: load_8x16B_global_live_out:
+; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB12_2
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %use
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1250-GISEL-NEXT: .LBB12_2: ; %exit
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ br i1 %cond, label %use, label %exit
+
+use:
+ store <4 x i32> %val, ptr addrspace(1) %out
+ br label %exit
+
+exit:
+ ret void
+}
+
+!0 = !{!""}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
index ce41cfc7b2c2494..243d6623d09a990 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -48,6 +48,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, pt
ret void
}
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x i32>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x i32> %a0, i32 0
+ %num2 = extractelement <2 x i32> %a0, i32 1
+ %res = add i32 %num1, %num2
+ store i32 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index 6d0f7cb4a6c4e34..59190c85c3605a8 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -48,6 +48,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, pt
ret void
}
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic <2 x i32>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+ %num1 = extractelement <2 x i32> %a0, i32 0
+ %num2 = extractelement <2 x i32> %a0, i32 1
+ %res = add i32 %num1, %num2
+ store i32 %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
index 855091bb8a00537..f17be52794b6cdd 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -32,6 +32,30 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
ret void
}
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i32> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
index 3de24b553f793ae..4da8cd2c8e2c075 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -32,6 +32,30 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
ret void
}
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic <2 x i32> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+ ret void
+}
+
define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
>From abe7f7cacf53309be8af8c0de5f0b7cc24e234fd Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 6 Oct 2026 14:53:38 -0400
Subject: [PATCH 2/2] Fixes
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 +
.../llvm.amdgcn.cooperative.atomic-basic.ll | 212 ++++++++++++-
.../llvm.amdgcn.cooperative.atomic-vector.ll | 286 ------------------
3 files changed, 214 insertions(+), 290 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 64eb21d4c2675eb..c54d895f1992287 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1443,18 +1443,24 @@ static unsigned getIntrMemWidth(unsigned IntrID) {
case Intrinsic::amdgcn_global_load_async_to_lds_b32:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
case Intrinsic::amdgcn_global_store_async_from_lds_b32:
+ case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
+ case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
case Intrinsic::amdgcn_flat_load_monitor_b32:
case Intrinsic::amdgcn_global_load_monitor_b32:
return 32;
case Intrinsic::amdgcn_global_load_async_to_lds_b64:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
case Intrinsic::amdgcn_global_store_async_from_lds_b64:
+ case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
+ case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
case Intrinsic::amdgcn_flat_load_monitor_b64:
case Intrinsic::amdgcn_global_load_monitor_b64:
return 64;
case Intrinsic::amdgcn_global_load_async_to_lds_b128:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
case Intrinsic::amdgcn_global_store_async_from_lds_b128:
+ case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
+ case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
case Intrinsic::amdgcn_flat_load_monitor_b128:
case Intrinsic::amdgcn_global_load_monitor_b128:
return 128;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-basic.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-basic.ll
index 353e96a5dab51d2..3ea1c8d0d2f8310 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-basic.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-basic.ll
@@ -48,12 +48,216 @@ entry:
ret void
}
+; global pointers, and uses of the loaded value that query known bits or sign bits
+
+define void @test_store_16x8B_global(ptr addrspace(1) noundef %addr, <2 x i32> noundef %val) {
+; GCN-LABEL: test_store_16x8B_global:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ tail call void @llvm.amdgcn.cooperative.atomic.store.16x8B.p1(ptr addrspace(1) %addr, <2 x i32> %val, i32 0, metadata !0)
+ ret void
+}
+
+define void @test_store_8x16B_global(ptr addrspace(1) noundef %addr, <4 x i32> noundef %val) {
+; GCN-LABEL: test_store_8x16B_global:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_store_b128 v[0:1], v[2:5], off scope:SCOPE_SYS
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ tail call void @llvm.amdgcn.cooperative.atomic.store.8x16B.p1(ptr addrspace(1) %addr, <4 x i32> %val, i32 0, metadata !0)
+ ret void
+}
+
+define i32 @test_load_16x8B_extract_and(ptr addrspace(1) noundef readonly %addr) {
+; GCN-LABEL: test_load_16x8B_extract_and:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GCN-NEXT: s_wait_loadcnt 0x0
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %0 = tail call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <2 x i32> %0, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i32 @test_load_8x16B_extract_and(ptr addrspace(1) noundef readonly %addr) {
+; GCN-LABEL: test_load_8x16B_extract_and:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GCN-NEXT: s_wait_loadcnt 0x0
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %0 = tail call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %0, i32 0
+ %and = and i32 %elt, 255
+ ret i32 %and
+}
+
+define i64 @test_load_8x16B_bitcast_v2i64_and(ptr addrspace(1) noundef readonly %addr) {
+; GCN-LABEL: test_load_8x16B_bitcast_v2i64_and:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GCN-NEXT: s_mov_b32 s0, 0xff
+; GCN-NEXT: s_wait_loadcnt 0x0
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %0 = tail call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %cast = bitcast <4 x i32> %0 to <2 x i64>
+ %elt = extractelement <2 x i64> %cast, i64 0
+ %and = and i64 %elt, 255
+ ret i64 %and
+}
+
+define half @test_load_8x16B_bitcast_v8f16(ptr addrspace(1) noundef readonly %addr) {
+; GCN-LABEL: test_load_8x16B_bitcast_v8f16:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GCN-NEXT: s_wait_loadcnt 0x0
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %0 = tail call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %cast = bitcast <4 x i32> %0 to <8 x half>
+ %lo = extractelement <8 x half> %cast, i32 0
+ %hi = extractelement <8 x half> %cast, i32 1
+ %add = fadd half %lo, %hi
+ ret half %add
+}
+
+define i32 @test_load_8x16B_sext_inreg(ptr addrspace(1) noundef readonly %addr) {
+; GCN-LABEL: test_load_8x16B_sext_inreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GCN-NEXT: s_wait_loadcnt 0x0
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: v_bfe_i32 v0, v3, 0, 16
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %0 = tail call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ %elt = extractelement <4 x i32> %0, i32 3
+ %shl = shl i32 %elt, 16
+ %ashr = ashr i32 %shl, 16
+ ret i32 %ashr
+}
+
+define void @test_load_8x16B_live_out(ptr addrspace(1) noundef readonly %addr, ptr addrspace(1) noundef %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: test_load_8x16B_live_out:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB10_2
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1250-SDAG-NEXT: .LBB10_2: ; %exit
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1251-SDAG-LABEL: test_load_8x16B_live_out:
+; GFX1251-SDAG: ; %bb.0: ; %entry
+; GFX1251-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1251-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1251-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1251-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v0
+; GFX1251-SDAG-NEXT: s_cbranch_execz .LBB10_2
+; GFX1251-SDAG-NEXT: ; %bb.1: ; %use
+; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1251-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1251-SDAG-NEXT: .LBB10_2: ; %exit
+; GFX1251-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1251-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1251-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: test_load_8x16B_live_out:
+; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB10_2
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %use
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1250-GISEL-NEXT: .LBB10_2: ; %exit
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1251-GISEL-LABEL: test_load_8x16B_live_out:
+; GFX1251-GISEL: ; %bb.0: ; %entry
+; GFX1251-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1251-GISEL-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1251-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 1, v4
+; GFX1251-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v0
+; GFX1251-GISEL-NEXT: s_cbranch_execz .LBB10_2
+; GFX1251-GISEL-NEXT: ; %bb.1: ; %use
+; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1251-GISEL-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GFX1251-GISEL-NEXT: .LBB10_2: ; %exit
+; GFX1251-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1251-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1251-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %0 = tail call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+ br i1 %cond, label %use, label %exit
+
+use:
+ store <4 x i32> %0, ptr addrspace(1) %out
+ br label %exit
+
+exit:
+ ret void
+}
+
!0 = !{ !"" }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1250: {{.*}}
-; GFX1250-GISEL: {{.*}}
-; GFX1250-SDAG: {{.*}}
; GFX1251: {{.*}}
-; GFX1251-GISEL: {{.*}}
-; GFX1251-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
deleted file mode 100644
index cbadd490b89ba59..000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
+++ /dev/null
@@ -1,286 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
-
-; 64-bit and 128-bit cooperative atomics on global and flat pointers, including
-; uses of the loaded value that query known bits and sign bits, in the same
-; block and across blocks.
-
-define <2 x i32> @load_16x8B_global(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_16x8B_global:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- ret <2 x i32> %val
-}
-
-define <4 x i32> @load_8x16B_global(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_8x16B_global:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- ret <4 x i32> %val
-}
-
-define void @store_16x8B_global(ptr addrspace(1) %addr, <2 x i32> %val) {
-; GFX1250-LABEL: store_16x8B_global:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- call void @llvm.amdgcn.cooperative.atomic.store.16x8B.p1(ptr addrspace(1) %addr, <2 x i32> %val, i32 0, metadata !0)
- ret void
-}
-
-define void @store_8x16B_global(ptr addrspace(1) %addr, <4 x i32> %val) {
-; GFX1250-LABEL: store_8x16B_global:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- call void @llvm.amdgcn.cooperative.atomic.store.8x16B.p1(ptr addrspace(1) %addr, <4 x i32> %val, i32 0, metadata !0)
- ret void
-}
-
-define i32 @load_16x8B_global_extract_and(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_16x8B_global_extract_and:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- %elt = extractelement <2 x i32> %val, i32 0
- %and = and i32 %elt, 255
- ret i32 %and
-}
-
-define i32 @load_16x8B_flat_extract_and(ptr %addr) {
-; GFX1250-LABEL: load_16x8B_flat_extract_and:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p0(ptr %addr, i32 0, metadata !0)
- %elt = extractelement <2 x i32> %val, i32 0
- %and = and i32 %elt, 255
- ret i32 %and
-}
-
-define i32 @load_8x16B_global_extract_and(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_8x16B_global_extract_and:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- %elt = extractelement <4 x i32> %val, i32 0
- %and = and i32 %elt, 255
- ret i32 %and
-}
-
-define i32 @load_8x16B_flat_extract_and(ptr %addr) {
-; GFX1250-LABEL: load_8x16B_flat_extract_and:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v[0:1] scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p0(ptr %addr, i32 0, metadata !0)
- %elt = extractelement <4 x i32> %val, i32 0
- %and = and i32 %elt, 255
- ret i32 %and
-}
-
-define i64 @load_8x16B_global_bitcast_v2i64_and(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_8x16B_global_bitcast_v2i64_and:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_mov_b32 s0, 0xff
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- %cast = bitcast <4 x i32> %val to <2 x i64>
- %elt = extractelement <2 x i64> %cast, i64 0
- %and = and i64 %elt, 255
- ret i64 %and
-}
-
-define i32 @load_16x8B_global_sext_inreg(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_16x8B_global_sext_inreg:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_bfe_i32 v0, v1, 0, 16
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- %elt = extractelement <2 x i32> %val, i32 1
- %shl = shl i32 %elt, 16
- %ashr = ashr i32 %shl, 16
- ret i32 %ashr
-}
-
-define i32 @load_8x16B_global_sext_inreg(ptr addrspace(1) %addr) {
-; GFX1250-LABEL: load_8x16B_global_sext_inreg:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_bfe_i32 v0, v3, 0, 16
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- %elt = extractelement <4 x i32> %val, i32 3
- %shl = shl i32 %elt, 16
- %ashr = ashr i32 %shl, 16
- ret i32 %ashr
-}
-
-define void @load_16x8B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
-; GFX1250-SDAG-LABEL: load_16x8B_global_live_out:
-; GFX1250-SDAG: ; %bb.0: ; %entry
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
-; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 1, v4
-; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v4
-; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB11_2
-; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
-; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GFX1250-SDAG-NEXT: .LBB11_2: ; %exit
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-GISEL-LABEL: load_16x8B_global_live_out:
-; GFX1250-GISEL: ; %bb.0: ; %entry
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 1, v4
-; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v4
-; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB11_2
-; GFX1250-GISEL-NEXT: ; %bb.1: ; %use
-; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GFX1250-GISEL-NEXT: .LBB11_2: ; %exit
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- br i1 %cond, label %use, label %exit
-
-use:
- store <2 x i32> %val, ptr addrspace(1) %out
- br label %exit
-
-exit:
- ret void
-}
-
-define void @load_8x16B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
-; GFX1250-SDAG-LABEL: load_8x16B_global_live_out:
-; GFX1250-SDAG: ; %bb.0: ; %entry
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 1, v4
-; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_cmpx_eq_u32_e32 1, v0
-; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB12_2
-; GFX1250-SDAG-NEXT: ; %bb.1: ; %use
-; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
-; GFX1250-SDAG-NEXT: .LBB12_2: ; %exit
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-GISEL-LABEL: load_8x16B_global_live_out:
-; GFX1250-GISEL: ; %bb.0: ; %entry
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 1, v4
-; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e32 0, v0
-; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB12_2
-; GFX1250-GISEL-NEXT: ; %bb.1: ; %use
-; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: global_store_b128 v[2:3], v[6:9], off
-; GFX1250-GISEL-NEXT: .LBB12_2: ; %exit
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
- br i1 %cond, label %use, label %exit
-
-use:
- store <4 x i32> %val, ptr addrspace(1) %out
- br label %exit
-
-exit:
- ret void
-}
-
-!0 = !{!""}
More information about the llvm-commits
mailing list