[llvm] [AMDGPU] Use vector memory types for 64/128-bit cooperative atomics (PR #229501)

Gheorghe-Teodor Bercea via llvm-commits llvm-commits at lists.llvm.org
Tue Oct 6 10:48:33 PDT 2026


https://github.com/doru1004 created https://github.com/llvm/llvm-project/pull/229501

The 16x8B and 8x16B cooperative atomic intrinsics had integer memory types (`i64`/`i128`) that didn't match their vector values, which broke value tracking on the loaded value and could crash the compiler. This patch uses the value type as the memory type and updates the selection patterns to match. Codegen for existing cooperative atomics is unchanged.





>From cc6c707183874c32446d67bc186b45b9e0c924f0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 6 Oct 2026 13:11:51 -0400
Subject: [PATCH] Use vector memory types for 64/128-bit cooperative atomics

---
 .../include/llvm/Target/TargetSelectionDAG.td |  24 +-
 llvm/lib/Target/AMDGPU/AMDGPUInstructions.td  |  12 +-
 llvm/lib/Target/AMDGPU/FLATInstructions.td    |  16 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  12 +-
 .../llvm.amdgcn.cooperative.atomic-vector.ll  | 286 ++++++++++++++++++
 llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll  |  40 +++
 .../test/CodeGen/AMDGPU/load-atomic-global.ll |  40 +++
 llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll |  24 ++
 .../CodeGen/AMDGPU/store-atomic-global.ll     |  24 ++
 9 files changed, 451 insertions(+), 27 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll

diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 4f8a4987ff838a..352fbcf72a22ba 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -2165,6 +2165,13 @@ def atomic_load_64 :
   let MemoryVT = i64;
 }
 
+def atomic_load_64_v2i32 :
+  PatFrag<(ops node:$ptr),
+          (atomic_load node:$ptr)> {
+  let IsAtomic = true;
+  let MemoryVT = v2i32;
+}
+
 def atomic_load_128_v2i64 :
   PatFrag<(ops node:$ptr),
           (atomic_load node:$ptr)> {
@@ -2203,12 +2210,6 @@ def atomic_load_nonext_64 :
   let MemoryVT = i64;
 }
 
-def atomic_load_nonext_128 :
-  PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> {
-  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
-  let MemoryVT = i128;
-}
-
 def atomic_load_zext_8 :
   PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
   let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
@@ -2440,11 +2441,18 @@ def atomic_store_64 :
   let MemoryVT = i64;
 }
 
-def atomic_store_128 :
+def atomic_store_64_v2i32 :
   PatFrag<(ops node:$val, node:$ptr),
           (atomic_store node:$val, node:$ptr)> {
   let IsAtomic = true;
-  let MemoryVT = i128;
+  let MemoryVT = v2i32;
+}
+
+def atomic_store_128_v4i32 :
+  PatFrag<(ops node:$val, node:$ptr),
+          (atomic_store node:$val, node:$ptr)> {
+  let IsAtomic = true;
+  let MemoryVT = v4i32;
 }
 
 // Matches store or atomic_store, no alignment requirement.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 434ad35f734995..6833020be0aa41 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -540,7 +540,11 @@ def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64
   let IsAtomic = 1;
 }
 
-def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> {
+def atomic_load_64_v2i32_#as : PatFrag<(ops node:$ptr), (atomic_load_64_v2i32 node:$ptr)> {
+  let IsAtomic = 1;
+}
+
+def atomic_load_128_v4i32_#as : PatFrag<(ops node:$ptr), (atomic_load_128_v4i32 node:$ptr)> {
   let IsAtomic = 1;
 }
 
@@ -608,8 +612,10 @@ def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
                                   (atomic_store_32 node:$val, node:$ptr)>;
 def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
                                   (atomic_store_64 node:$val, node:$ptr)>;
-def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr),
-                                   (atomic_store_128 node:$val, node:$ptr)>;
+def atomic_store_64_v2i32_#as : PatFrag<(ops node:$val, node:$ptr),
+                                        (atomic_store_64_v2i32 node:$val, node:$ptr)>;
+def atomic_store_128_v4i32_#as : PatFrag<(ops node:$val, node:$ptr),
+                                         (atomic_store_128_v4i32 node:$val, node:$ptr)>;
 } // End let IsAtomic = 1, AddressSpaces = ...
 } // End foreach as
 
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index f241f6677d22b2..fb3ea1c0055a33 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2111,8 +2111,8 @@ defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
 defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>;
 defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;
 defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;
-defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_64_v2i32_flat, v2i32>;
+defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_128_v4i32_flat, v4i32>;
 
 defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
 defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
@@ -2137,9 +2137,9 @@ defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;
 defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
 defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>;
 defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
-defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_v2i32_flat, v2i32>;
 defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>;
-defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;
+defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_v4i32_flat, v4i32>;
 defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
 defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
 
@@ -2294,9 +2294,9 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_v2i32_global, v2i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_128_v4i32_global, v4i32>;
 
 defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
@@ -2344,8 +2344,8 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_v2i32_global, v2i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_v4i32_global, v4i32>;
 
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 31ab77d5e900bb..64eb21d4c2675e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1443,24 +1443,18 @@ static unsigned getIntrMemWidth(unsigned IntrID) {
   case Intrinsic::amdgcn_global_load_async_to_lds_b32:
   case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
   case Intrinsic::amdgcn_global_store_async_from_lds_b32:
-  case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
-  case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
   case Intrinsic::amdgcn_flat_load_monitor_b32:
   case Intrinsic::amdgcn_global_load_monitor_b32:
     return 32;
   case Intrinsic::amdgcn_global_load_async_to_lds_b64:
   case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
   case Intrinsic::amdgcn_global_store_async_from_lds_b64:
-  case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
-  case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
   case Intrinsic::amdgcn_flat_load_monitor_b64:
   case Intrinsic::amdgcn_global_load_monitor_b64:
     return 64;
   case Intrinsic::amdgcn_global_load_async_to_lds_b128:
   case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
   case Intrinsic::amdgcn_global_store_async_from_lds_b128:
-  case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
-  case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
   case Intrinsic::amdgcn_flat_load_monitor_b128:
   case Intrinsic::amdgcn_global_load_monitor_b128:
     return 128;
@@ -1781,7 +1775,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
   case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
   case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
+    // These are lowered to generic atomic loads and stores, which require the
+    // memory type to match the value type.
+    Info.memVT = MVT::getVT(CI.getType());
     Info.ptrVal = CI.getOperand(0);
     Info.align.reset();
     Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
@@ -1794,7 +1790,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
   case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
   case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
     Info.opc = ISD::INTRINSIC_VOID;
-    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
+    Info.memVT = MVT::getVT(CI.getArgOperand(1)->getType());
     Info.ptrVal = CI.getArgOperand(0);
     Info.align.reset();
     Info.flags = (MachineMemOperand::MOStore | MOCooperative);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
new file mode 100644
index 00000000000000..cbadd490b89ba5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cooperative.atomic-vector.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+; 64-bit and 128-bit cooperative atomics on global and flat pointers, including
+; uses of the loaded value that query known bits and sign bits, in the same
+; block and across blocks.
+
+define <2 x i32> @load_16x8B_global(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  ret <2 x i32> %val
+}
+
+define <4 x i32> @load_8x16B_global(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  ret <4 x i32> %val
+}
+
+define void @store_16x8B_global(ptr addrspace(1) %addr, <2 x i32> %val) {
+; GFX1250-LABEL: store_16x8B_global:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  call void @llvm.amdgcn.cooperative.atomic.store.16x8B.p1(ptr addrspace(1) %addr, <2 x i32> %val, i32 0, metadata !0)
+  ret void
+}
+
+define void @store_8x16B_global(ptr addrspace(1) %addr, <4 x i32> %val) {
+; GFX1250-LABEL: store_8x16B_global:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  call void @llvm.amdgcn.cooperative.atomic.store.8x16B.p1(ptr addrspace(1) %addr, <4 x i32> %val, i32 0, metadata !0)
+  ret void
+}
+
+define i32 @load_16x8B_global_extract_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global_extract_and:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  %elt = extractelement <2 x i32> %val, i32 0
+  %and = and i32 %elt, 255
+  ret i32 %and
+}
+
+define i32 @load_16x8B_flat_extract_and(ptr %addr) {
+; GFX1250-LABEL: load_16x8B_flat_extract_and:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p0(ptr %addr, i32 0, metadata !0)
+  %elt = extractelement <2 x i32> %val, i32 0
+  %and = and i32 %elt, 255
+  ret i32 %and
+}
+
+define i32 @load_8x16B_global_extract_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_extract_and:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  %elt = extractelement <4 x i32> %val, i32 0
+  %and = and i32 %elt, 255
+  ret i32 %and
+}
+
+define i32 @load_8x16B_flat_extract_and(ptr %addr) {
+; GFX1250-LABEL: load_8x16B_flat_extract_and:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b128 v[0:3], v[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p0(ptr %addr, i32 0, metadata !0)
+  %elt = extractelement <4 x i32> %val, i32 0
+  %and = and i32 %elt, 255
+  ret i32 %and
+}
+
+define i64 @load_8x16B_global_bitcast_v2i64_and(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_bitcast_v2i64_and:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_mov_b32 s0, 0xff
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  %cast = bitcast <4 x i32> %val to <2 x i64>
+  %elt = extractelement <2 x i64> %cast, i64 0
+  %and = and i64 %elt, 255
+  ret i64 %and
+}
+
+define i32 @load_16x8B_global_sext_inreg(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_16x8B_global_sext_inreg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v0, v1, 0, 16
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  %elt = extractelement <2 x i32> %val, i32 1
+  %shl = shl i32 %elt, 16
+  %ashr = ashr i32 %shl, 16
+  ret i32 %ashr
+}
+
+define i32 @load_8x16B_global_sext_inreg(ptr addrspace(1) %addr) {
+; GFX1250-LABEL: load_8x16B_global_sext_inreg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v0, v3, 0, 16
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  %elt = extractelement <4 x i32> %val, i32 3
+  %shl = shl i32 %elt, 16
+  %ashr = ashr i32 %shl, 16
+  ret i32 %ashr
+}
+
+define void @load_16x8B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: load_16x8B_global_live_out:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX1250-SDAG-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    v_cmpx_eq_u32_e32 1, v4
+; GFX1250-SDAG-NEXT:    s_cbranch_execz .LBB11_2
+; GFX1250-SDAG-NEXT:  ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT:  .LBB11_2: ; %exit
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: load_16x8B_global_live_out:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX1250-GISEL-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    v_cmpx_ne_u32_e32 0, v4
+; GFX1250-GISEL-NEXT:    s_cbranch_execz .LBB11_2
+; GFX1250-GISEL-NEXT:  ; %bb.1: ; %use
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-GISEL-NEXT:  .LBB11_2: ; %exit
+; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <2 x i32> @llvm.amdgcn.cooperative.atomic.load.16x8B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  br i1 %cond, label %use, label %exit
+
+use:
+  store <2 x i32> %val, ptr addrspace(1) %out
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @load_8x16B_global_live_out(ptr addrspace(1) %addr, ptr addrspace(1) %out, i1 %cond) {
+; GFX1250-SDAG-LABEL: load_8x16B_global_live_out:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v0, 1, v4
+; GFX1250-SDAG-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    v_cmpx_eq_u32_e32 1, v0
+; GFX1250-SDAG-NEXT:    s_cbranch_execz .LBB12_2
+; GFX1250-SDAG-NEXT:  ; %bb.1: ; %use
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b128 v[2:3], v[6:9], off
+; GFX1250-SDAG-NEXT:  .LBB12_2: ; %exit
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: load_8x16B_global_live_out:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT:    global_load_b128 v[6:9], v[0:1], off scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v0, 1, v4
+; GFX1250-GISEL-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    v_cmpx_ne_u32_e32 0, v0
+; GFX1250-GISEL-NEXT:    s_cbranch_execz .LBB12_2
+; GFX1250-GISEL-NEXT:  ; %bb.1: ; %use
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b128 v[2:3], v[6:9], off
+; GFX1250-GISEL-NEXT:  .LBB12_2: ; %exit
+; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    s_set_pc_i64 s[30:31]
+entry:
+  %val = call <4 x i32> @llvm.amdgcn.cooperative.atomic.load.8x16B.p1(ptr addrspace(1) %addr, i32 0, metadata !0)
+  br i1 %cond, label %use, label %exit
+
+use:
+  store <4 x i32> %val, ptr addrspace(1) %out
+  br label %exit
+
+exit:
+  ret void
+}
+
+!0 = !{!""}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
index ce41cfc7b2c249..243d6623d09a99 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -48,6 +48,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, pt
   ret void
 }
 
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
+; GFX12-NEXT:    s_endpgm
+  %a0 = load atomic <2 x i32>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
+  %num1 = extractelement <2 x i32> %a0, i32 0
+  %num2 = extractelement <2 x i32> %a0, i32 1
+  %res = add i32 %num1, %num2
+  store i32 %res, ptr addrspace(1) %out, align 4
+  ret void
+}
+
 define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
 ; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
 ; GFX9:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index 6d0f7cb4a6c4e3..59190c85c3605a 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -48,6 +48,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, pt
   ret void
 }
 
+define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
+; GFX12-NEXT:    s_endpgm
+  %a0 = load atomic <2 x i32>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
+  %num1 = extractelement <2 x i32> %a0, i32 0
+  %num2 = extractelement <2 x i32> %a0, i32 1
+  %res = add i32 %num1, %num2
+  store i32 %res, ptr addrspace(1) %out, align 4
+  ret void
+}
+
 define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
 ; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
 ; GFX9:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
index 855091bb8a0053..f17be52794b6cd 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -32,6 +32,30 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
   ret void
 }
 
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT:    s_endpgm
+  store atomic <2 x i32> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 8
+  ret void
+}
+
 define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
 ; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
 ; GFX9:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
index 3de24b553f793a..4da8cd2c8e2c07 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -32,6 +32,30 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
   ret void
 }
 
+define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT:    s_endpgm
+  store atomic <2 x i32> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
+  ret void
+}
+
 define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
 ; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
 ; GFX9:       ; %bb.0:



More information about the llvm-commits mailing list