[clang] [llvm] [mlir] [AMDGPU] Fix missing cache bypass scope on seq_cst buffer fat pointer atomics (PR #213585)
Arseniy Obolenskiy via cfe-commits
cfe-commits at lists.llvm.org
Mon Aug 10 21:04:09 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/213585
>From 1d254511c60af3c1cb7575a5caa83635412a4915 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 31 Jul 2026 17:18:47 +0200
Subject: [PATCH 1/6] [AMDGPU] Fix missing cache bypass scope on seq_cst buffer
fat pointer atomics
Buffer atomic/load/store intrinsics have no scope operand, so the
scope was lost during lowering
Stash it as metadata and recover it in SIISelLowering so SIMemoryLegalizer sets cache bypass bits
---
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 16 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 27 +++
.../AMDGPU/memory-legalizer-buffer-atomics.ll | 155 +++++++++++++++++-
3 files changed, 196 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index dd2f7075c8df2..bb12289821749 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1566,6 +1566,10 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
+ // Buffer intrinsics have no ordering/scope operand, so stash it as
+ // metadata for SIISelLowering::getTgtMemIntrinsic to recover.
+ void attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
+ SyncScope::ID SSID);
Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
Align Alignment, AtomicOrdering Order,
bool IsVolatile, SyncScope::ID SSID);
@@ -1872,6 +1876,16 @@ void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A));
}
+void SplitPtrStructs::attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
+ SyncScope::ID SSID) {
+ if (Order == AtomicOrdering::NotAtomic)
+ return;
+ LLVMContext &Ctx = Call->getContext();
+ StringRef Scope = Ctx.getSyncScopeName(SSID).value_or("");
+ Call->setMetadata("amdgpu.buffer.atomic.scope",
+ MDNode::get(Ctx, MDString::get(Ctx, Scope)));
+}
+
void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
SyncScope::ID SSID) {
switch (Order) {
@@ -2023,6 +2037,7 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(IID, Ty, Args);
copyMetadata(Call, I);
+ attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, Alignment, Arg ? 1 : 0);
Call->takeName(I);
@@ -2093,6 +2108,7 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
IRB.getInt32(0), IRB.getInt32(Aux)});
copyMetadata(Call, &AI);
+ attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 403f5d7476dfc..dc9725a7c2de9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1416,6 +1416,16 @@ static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
return CI.getContext().getOrInsertSyncScopeID(Scope);
}
+// Recovers the scope AMDGPULowerBufferFatPointers stashed as metadata,
+// since buffer intrinsics have no scope operand of their own.
+static std::optional<SyncScope::ID> parseAtomicScopeMD(const CallBase &CI) {
+ MDNode *ScopeMD = CI.getMetadata("amdgpu.buffer.atomic.scope");
+ if (!ScopeMD)
+ return std::nullopt;
+ StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
+ return CI.getContext().getOrInsertSyncScopeID(Scope);
+}
+
void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
const CallBase &CI,
MachineFunction &MF,
@@ -1503,6 +1513,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = getValueType(MF.getDataLayout(), DataTy);
Info.flags = Flags | MachineMemOperand::MOStore;
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so SIMemoryLegalizer::expandStore sets cache
+ // bypass bits, since atomic and plain buffer stores look identical.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
} else {
// Atomic, NoReturn Sampler or prefetch
Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
@@ -1521,6 +1537,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// XXX - Should this be volatile without known ordering?
Info.flags |= MachineMemOperand::MOVolatile;
Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so expandAtomicCmpxchgOrRmw sets cache
+ // bypass bits.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
}
break;
case Intrinsic::amdgcn_raw_buffer_load_lds:
@@ -1561,6 +1583,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags = Flags | MachineMemOperand::MOLoad;
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so expandLoad sets cache bypass bits.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
Infos.push_back(Info);
return;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
index 1cf8ef7a918f4..6523c168dd50c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
@@ -22,7 +22,7 @@ define void @buffer_fat_ptr_agent_atomic_add_noret_i32(ptr addrspace(7) inreg %p
; GFX1250-NEXT: s_mov_b32 s3, s8
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_DEV
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%unused = atomicrmw add ptr addrspace(7) %ptr, i32 %val syncscope("agent") monotonic
ret void
@@ -49,13 +49,164 @@ define i32 @buffer_fat_ptr_agent_atomic_add_ret_i32(ptr addrspace(7) inreg %ptr,
; GFX1250-NEXT: s_mov_b32 s3, s8
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = atomicrmw add ptr addrspace(7) %ptr, i32 %val syncscope("agent") monotonic
ret i32 %ret
}
+define void @buffer_fat_ptr_seq_cst_atomic_add_noret_i32(ptr addrspace(7) inreg %ptr, i32 %val) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_atomic_add_noret_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %unused = atomicrmw add ptr addrspace(7) %ptr, i32 %val seq_cst
+ ret void
+}
+
+define i32 @buffer_fat_ptr_seq_cst_cmpxchg_i32(ptr addrspace(7) inreg %ptr, i32 %cmp, i32 %new) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_cmpxchg_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: scratch_store_b32 off, v1, s32 nv ; 4-byte Folded Spill
+; GFX1250-NEXT: v_mov_b32_e32 v2, v0
+; GFX1250-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_mov_b32_e32 v1, v2
+; GFX1250-NEXT: v_mov_b32_e32 v2, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %pair = cmpxchg ptr addrspace(7) %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+ %ret = extractvalue { i32, i1 } %pair, 0
+ ret i32 %ret
+}
+
+define i32 @buffer_fat_ptr_seq_cst_load_i32(ptr addrspace(7) inreg %ptr) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_load_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s16
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %ret = load atomic i32, ptr addrspace(7) %ptr seq_cst, align 4
+ ret i32 %ret
+}
+
+define void @buffer_fat_ptr_seq_cst_store_i32(ptr addrspace(7) inreg %ptr, i32 %val) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_store_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ store atomic i32 %val, ptr addrspace(7) %ptr seq_cst, align 4
+ ret void
+}
+
define void @raw_buffer_atomic_add_v2f16_noret(<2 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
; GFX1250-LABEL: raw_buffer_atomic_add_v2f16_noret:
; GFX1250: ; %bb.0:
>From 02209a898ab435f7c4f28c04aacb4e689a84f3bd Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 5 Aug 2026 06:15:48 +0200
Subject: [PATCH 2/6] doc
---
llvm/docs/AMDGPUUsage.rst | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 87ba811e1a686..a2f1d878cb783 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2484,6 +2484,28 @@ whether to skip the DPP optimization.
; Many expected active lanes: the DPP optimization is not skipped.
%old1 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 32}
+.. _amdgpu_buffer_atomic_scope:
+
+'``amdgpu.buffer.atomic.scope``' Metadata
+-------------------------------------------------
+
+Internal metadata attached by ``AMDGPULowerBufferFatPointers`` to the
+buffer intrinsics it emits for atomic, and ``seq_cst`` load or store,
+accesses through ptr addrspace(7) (buffer fat pointer). Buffer
+intrinsics have no scope operand of their own, so this metadata carries
+the :ref:`synchronization scope<amdgpu-memory-scopes>` name of the
+original memory operation through to instruction selection, where
+``SITargetLowering::getTgtMemIntrinsic`` recovers it so
+``SIMemoryLegalizer`` can set the correct cache bypass bits. This
+metadata is only produced and consumed internally by the AMDGPU backend
+and is not intended to be written by front ends.
+
+.. code-block:: llvm
+
+ %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0), !amdgpu.buffer.atomic.scope !0
+
+ !0 = !{!"agent"}
+
LLVM IR Attributes
==================
>From 0cdf82a3c04379316404a995fca772fb334a06a0 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 6 Aug 2026 10:55:19 +0200
Subject: [PATCH 3/6] address comment
---
llvm/docs/AMDGPUUsage.rst | 23 -
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 55 +-
llvm/lib/IR/AutoUpgrade.cpp | 157 +++
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 24 +-
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 44 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 33 +-
.../amdgpu-buffer-atomic-scope-upgrade.ll | 33 +
.../atomic_optimizations_mul_one.ll | 12 +-
.../AMDGPU/lower-buffer-fat-pointers-calls.ll | 4 +-
.../lower-buffer-fat-pointers-constants.ll | 4 +-
...ointers-contents-legalization-alignment.ll | 667 +++++-----
...ffer-fat-pointers-contents-legalization.ll | 255 ++--
.../lower-buffer-fat-pointers-mem-transfer.ll | 1089 +++++++++--------
.../lower-buffer-fat-pointers-memops.ll | 54 +-
...fer-fat-pointers-unoptimized-debug-data.ll | 4 +-
.../FunctionAttrs/make-buffer-rsrc.ll | 4 +-
.../LICM/AMDGPU/buffer-rsrc-ptrs.ll | 20 +-
17 files changed, 1372 insertions(+), 1110 deletions(-)
create mode 100644 llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index a2f1d878cb783..50caa5ba7e265 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2484,29 +2484,6 @@ whether to skip the DPP optimization.
; Many expected active lanes: the DPP optimization is not skipped.
%old1 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 32}
-.. _amdgpu_buffer_atomic_scope:
-
-'``amdgpu.buffer.atomic.scope``' Metadata
--------------------------------------------------
-
-Internal metadata attached by ``AMDGPULowerBufferFatPointers`` to the
-buffer intrinsics it emits for atomic, and ``seq_cst`` load or store,
-accesses through ptr addrspace(7) (buffer fat pointer). Buffer
-intrinsics have no scope operand of their own, so this metadata carries
-the :ref:`synchronization scope<amdgpu-memory-scopes>` name of the
-original memory operation through to instruction selection, where
-``SITargetLowering::getTgtMemIntrinsic`` recovers it so
-``SIMemoryLegalizer`` can set the correct cache bypass bits. This
-metadata is only produced and consumed internally by the AMDGPU backend
-and is not intended to be written by front ends.
-
-.. code-block:: llvm
-
- %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0), !amdgpu.buffer.atomic.scope !0
-
- !0 = !{!"agent"}
-
-
LLVM IR Attributes
==================
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 9c57b4d343fc6..b2862f1db0ea5 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1585,10 +1585,11 @@ class AMDGPURawPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
[AMDGPUBufferRsrcTy,// rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc,
+ llvm_i32_ty, // auxiliary data (imm, cachepolicy (bit 0 = glc,
// bit 1 = slc,
// bit 2 = dlc on gfx10+),
// swizzled buffer (bit 3 = swz))
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
def int_amdgcn_raw_ptr_atomic_buffer_load : AMDGPURawPtrAtomicBufferLoad;
@@ -1653,13 +1654,14 @@ class AMDGPUStructPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsi
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary/cachepolicy(imm):
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
// bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
// bit 3 = swz, bit 4 = scc (gfx90a)
// gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<0>>,
ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
@@ -1700,7 +1702,23 @@ class AMDGPURawPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntr
ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1>;
def int_amdgcn_raw_ptr_buffer_store_format : AMDGPURawPtrBufferStore<llvm_anyfloat_ty>;
-def int_amdgcn_raw_ptr_buffer_store : AMDGPURawPtrBufferStore;
+def int_amdgcn_raw_ptr_buffer_store : DefaultAttrsIntrinsic <
+ [],
+ [llvm_any_ty, // vdata(VGPR)
+ AMDGPUBufferRsrcTy, // rsrc(SGPR)
+ llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
+ llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
+ // bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
+ // bit 3 = swz, bit 4 = scc (gfx90a)
+ // gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
+ // gfx12+: bits [0-2] = th, bits [3-4] = scope,
+ // bit 6 = swz
+ // all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
+ [IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
+ ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
+ AMDGPURsrcIntrinsic<1>;
class AMDGPUStructBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
[],
@@ -1739,7 +1757,24 @@ class AMDGPUStructPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsI
ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1>;
def int_amdgcn_struct_ptr_buffer_store_format : AMDGPUStructPtrBufferStore;
-def int_amdgcn_struct_ptr_buffer_store : AMDGPUStructPtrBufferStore;
+def int_amdgcn_struct_ptr_buffer_store : DefaultAttrsIntrinsic <
+ [],
+ [llvm_any_ty, // vdata(VGPR)
+ AMDGPUBufferRsrcTy, // rsrc(SGPR)
+ llvm_i32_ty, // vindex(VGPR)
+ llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
+ llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
+ // bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
+ // bit 3 = swz, bit 4 = scc (gfx90a)
+ // gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
+ // gfx12+: bits [0-2] = th, bits [3-4] = scope,
+ // bit 6 = swz
+ // all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
+ [IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
+ ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
+ AMDGPURsrcIntrinsic<1>;
class AMDGPURawBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
[data_ty],
@@ -1783,7 +1818,8 @@ class AMDGPURawPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1, 0>;
@@ -1811,7 +1847,8 @@ def int_amdgcn_raw_ptr_buffer_atomic_cmpswap : Intrinsic<
AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<2>>,
ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<2, 0>;
@@ -1865,7 +1902,8 @@ class AMDGPUStructPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1, 0>;
@@ -1891,7 +1929,8 @@ def int_amdgcn_struct_ptr_buffer_atomic_cmpswap : Intrinsic<
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<2>>,
ImmArg<ArgIndex<6>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<2, 0>;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 4502759417c5a..865931fd1b1c0 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1400,6 +1400,88 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return true;
}
break;
+ // Legacy buffer atomic/store/atomic-load intrinsics missed the
+ // trailing syncscope metadata operand.
+ case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
+ if (F->arg_size() == 4) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
+ if (F->arg_size() == 5) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_raw_ptr_buffer_store:
+ if (F->arg_size() == 5) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_buffer_store:
+ if (F->arg_size() == 6) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
+ if (F->arg_size() == 5) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
+ if (F->arg_size() == 6) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
+ if (F->arg_size() == 6) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
+ if (F->arg_size() == 7) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
}
if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
@@ -4891,6 +4973,81 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
}
+ // Legacy buffer atomic/store/atomic-load intrinsics missed the trailing
+ // syncscope metadata operand. Append an empty MDNode, meaning "no scope
+ // info", matching the behavior of these calls before the operand existed.
+ auto UpgradeBufferScopeIntrinsicCall =
+ [](Function *F, CallBase *CI, IRBuilder<> &Builder,
+ ArrayRef<Type *> OverloadTys) -> Value * {
+ SmallVector<Value *, 8> Args(CI->args());
+ Args.push_back(MetadataAsValue::get(Builder.getContext(),
+ MDNode::get(Builder.getContext(), {})));
+
+ Function *NewDecl = Intrinsic::getOrInsertDeclaration(
+ F->getParent(), F->getIntrinsicID(), OverloadTys);
+
+ SmallVector<OperandBundleDef, 1> Bundles;
+ CI->getOperandBundlesAsDefs(Bundles);
+
+ auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
+ NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ NewCall->setCallingConv(CI->getCallingConv());
+ NewCall->setAttributes(CI->getAttributes());
+ NewCall->setDebugLoc(CI->getDebugLoc());
+ NewCall->copyMetadata(*CI);
+ NewCall->takeName(CI);
+ return NewCall;
+ };
+
+ switch (F->getIntrinsicID()) {
+ case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
+ case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
+ return UpgradeBufferScopeIntrinsicCall(F, CI, Builder,
+ {F->getReturnType()});
+ case Intrinsic::amdgcn_raw_ptr_buffer_store:
+ case Intrinsic::amdgcn_struct_ptr_buffer_store:
+ return UpgradeBufferScopeIntrinsicCall(F, CI, Builder,
+ {CI->getArgOperand(0)->getType()});
+ default:
+ break;
+ }
+
switch (F->getIntrinsicID()) {
default:
break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 6bf05f9808d78..3d115a1ed6551 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6690,11 +6690,17 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
unsigned ImmOffset;
+ // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // of the positional arguments counted below.
+ unsigned NumOperands = MI.getNumOperands();
+ if (MI.getOperand(NumOperands - 1).isMetadata())
+ --NumOperands;
+
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
// The struct intrinsic variants add one additional operand over raw.
- const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
+ const bool HasVIndex = NumOperands == NumVIndexOps;
Register VIndex;
int OpOffset = 0;
if (HasVIndex) {
@@ -6805,11 +6811,17 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
castBufferRsrcArgToV4I32(MI, B, 2 + OpOffset);
Register RSrc = MI.getOperand(2 + OpOffset).getReg();
+ // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // of the positional arguments counted below.
+ unsigned NumOperands = MI.getNumOperands();
+ if (MI.getOperand(NumOperands - 1).isMetadata())
+ --NumOperands;
+
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
// The struct intrinsic variants add one additional operand over raw.
- const bool HasVIndex = MI.getNumOperands() == NumVIndexOps + OpOffset;
+ const bool HasVIndex = NumOperands == NumVIndexOps + OpOffset;
Register VIndex;
if (HasVIndex) {
VIndex = MI.getOperand(3 + OpOffset).getReg();
@@ -7081,8 +7093,14 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
Register RSrc = MI.getOperand(3 + OpOffset).getReg();
const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
+ // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // of the positional arguments counted below.
+ unsigned NumOperands = MI.getNumOperands();
+ if (MI.getOperand(NumOperands - 1).isMetadata())
+ --NumOperands;
+
// The struct intrinsic variants add one additional operand over raw.
- const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
+ const bool HasVIndex = NumOperands == NumVIndexOps;
Register VIndex;
if (HasVIndex) {
VIndex = MI.getOperand(4 + OpOffset).getReg();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index bb12289821749..def10425dd165 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1566,10 +1566,9 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
- // Buffer intrinsics have no ordering/scope operand, so stash it as
- // metadata for SIISelLowering::getTgtMemIntrinsic to recover.
- void attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
- SyncScope::ID SSID);
+ // Build the trailing syncscope metadata argument for the buffer
+ // atomic/store/atomic-load intrinsics (empty string if not atomic).
+ Value *getScopeMDArg(AtomicOrdering Order, SyncScope::ID SSID);
Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
Align Alignment, AtomicOrdering Order,
bool IsVolatile, SyncScope::ID SSID);
@@ -1876,14 +1875,15 @@ void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A));
}
-void SplitPtrStructs::attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
- SyncScope::ID SSID) {
+Value *SplitPtrStructs::getScopeMDArg(AtomicOrdering Order,
+ SyncScope::ID SSID) {
+ LLVMContext &Ctx = IRB.getContext();
+ // An empty MDNode means not atomic. The system scope also has an empty
+ // name, so scope presence cannot be encoded by string emptiness alone.
if (Order == AtomicOrdering::NotAtomic)
- return;
- LLVMContext &Ctx = Call->getContext();
+ return MetadataAsValue::get(Ctx, MDNode::get(Ctx, {}));
StringRef Scope = Ctx.getSyncScopeName(SSID).value_or("");
- Call->setMetadata("amdgpu.buffer.atomic.scope",
- MDNode::get(Ctx, MDString::get(Ctx, Scope)));
+ return MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDString::get(Ctx, Scope)));
}
void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
@@ -1935,12 +1935,19 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
Aux |= AMDGPU::CPol::VOLATILE;
Args.push_back(IRB.getInt32(Aux));
+ // amdgcn_raw_ptr_buffer_load has no scope operand, unlike the atomic load,
+ // store, and RMW intrinsics used below.
+ bool NeedsScopeArg = true;
+
Intrinsic::ID IID = Intrinsic::not_intrinsic;
- if (isa<LoadInst>(I))
- IID = Order == AtomicOrdering::NotAtomic
- ? Intrinsic::amdgcn_raw_ptr_buffer_load
- : Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
- else if (isa<StoreInst>(I))
+ if (isa<LoadInst>(I)) {
+ if (Order == AtomicOrdering::NotAtomic) {
+ IID = Intrinsic::amdgcn_raw_ptr_buffer_load;
+ NeedsScopeArg = false;
+ } else {
+ IID = Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
+ }
+ } else if (isa<StoreInst>(I))
IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) {
switch (RMW->getOperation()) {
@@ -2035,9 +2042,11 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
}
}
+ if (NeedsScopeArg)
+ Args.push_back(getScopeMDArg(Order, SSID));
+
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(IID, Ty, Args);
copyMetadata(Call, I);
- attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, Alignment, Arg ? 1 : 0);
Call->takeName(I);
@@ -2106,9 +2115,8 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(
Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, Ty,
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
- IRB.getInt32(0), IRB.getInt32(Aux)});
+ IRB.getInt32(0), IRB.getInt32(Aux), getScopeMDArg(Order, SSID)});
copyMetadata(Call, &AI);
- attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index dc9725a7c2de9..69f50a966eaa3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1416,11 +1416,20 @@ static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
return CI.getContext().getOrInsertSyncScopeID(Scope);
}
-// Recovers the scope AMDGPULowerBufferFatPointers stashed as metadata,
-// since buffer intrinsics have no scope operand of their own.
-static std::optional<SyncScope::ID> parseAtomicScopeMD(const CallBase &CI) {
- MDNode *ScopeMD = CI.getMetadata("amdgpu.buffer.atomic.scope");
- if (!ScopeMD)
+// Buffer atomic/store/atomic-load intrinsics take the syncscope name as
+// their trailing metadata argument, an empty MDNode when the memory access
+// is not atomic (the system scope also has an empty name, so the MDNode
+// operand count, not string emptiness, distinguishes the two). Not every
+// intrinsic reaching this dispatch has that trailing argument, so callers
+// rely on this returning std::nullopt for those too.
+static std::optional<SyncScope::ID>
+parseBufferAtomicScopeArg(const CallBase &CI) {
+ auto *ScopeArg =
+ dyn_cast<MetadataAsValue>(CI.getArgOperand(CI.arg_size() - 1));
+ if (!ScopeArg)
+ return std::nullopt;
+ MDNode *ScopeMD = cast<MDNode>(ScopeArg->getMetadata());
+ if (ScopeMD->getNumOperands() == 0)
return std::nullopt;
StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
return CI.getContext().getOrInsertSyncScopeID(Scope);
@@ -1447,7 +1456,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
if (!IsSPrefetch) {
- auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
+ // Some buffer intrinsics have a trailing syncscope metadata arg after
+ // the cachepolicy immarg.
+ unsigned AuxIdx = CI.arg_size() - 1;
+ if (isa<MetadataAsValue>(CI.getArgOperand(AuxIdx)))
+ --AuxIdx;
+ auto *Aux = cast<ConstantInt>(CI.getArgOperand(AuxIdx));
if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
Flags |= MachineMemOperand::MOVolatile;
}
@@ -1513,7 +1527,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = getValueType(MF.getDataLayout(), DataTy);
Info.flags = Flags | MachineMemOperand::MOStore;
- if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ if (std::optional<SyncScope::ID> SSID = parseBufferAtomicScopeArg(CI)) {
// Pretend to be atomic so SIMemoryLegalizer::expandStore sets cache
// bypass bits, since atomic and plain buffer stores look identical.
Info.order = AtomicOrdering::Monotonic;
@@ -1537,7 +1551,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// XXX - Should this be volatile without known ordering?
Info.flags |= MachineMemOperand::MOVolatile;
Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
- if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ if (std::optional<SyncScope::ID> SSID =
+ parseBufferAtomicScopeArg(CI)) {
// Pretend to be atomic so expandAtomicCmpxchgOrRmw sets cache
// bypass bits.
Info.order = AtomicOrdering::Monotonic;
@@ -1583,7 +1598,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags = Flags | MachineMemOperand::MOLoad;
- if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ if (std::optional<SyncScope::ID> SSID = parseBufferAtomicScopeArg(CI)) {
// Pretend to be atomic so expandLoad sets cache bypass bits.
Info.order = AtomicOrdering::Monotonic;
Info.ssid = *SSID;
diff --git a/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll b/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
new file mode 100644
index 0000000000000..3f9550e026c8e
--- /dev/null
+++ b/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
@@ -0,0 +1,33 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+declare void @llvm.amdgcn.raw.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32, i32, ptr addrspace(8), i32, i32, i32 immarg)
+declare void @llvm.amdgcn.struct.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32, i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
+
+define void @test(i32 %v, ptr addrspace(8) %rsrc, float %f) {
+; CHECK-LABEL: define void @test(
+; CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+ call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %a = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %b = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %c = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %d = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %e = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %g = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+; CHECK: !0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index 4d08b2c582d2b..7fc4b3fbd9a50 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -322,7 +322,7 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: ret void
@@ -362,7 +362,7 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
@@ -422,7 +422,7 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: ret void
@@ -462,7 +462,7 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
@@ -523,7 +523,7 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB12]]
; IR: [[BB12]]:
; IR-NEXT: ret void
@@ -565,7 +565,7 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[TMP10:.*]], label %[[BB12:.*]]
; IR: [[TMP10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB12]]
; IR: [[BB12]]:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP11]], %[[TMP10]] ]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
index cf346a8fa2064..12cdaf3536f78 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
@@ -91,10 +91,10 @@ define void @caller(ptr addrspace(7) noundef nonnull %arg) {
; CHECK-NEXT: [[V_INT:%.*]] = or i160 [[TMP1]], [[V_INT_OFF]]
; CHECK-NEXT: [[V_INT_LEGAL:%.*]] = bitcast i160 [[V_INT]] to <5 x i32>
; CHECK-NEXT: [[V_INT_SLICE_0:%.*]] = shufflevector <5 x i32> [[V_INT_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[V_INT_SLICE_0]], ptr addrspace(8) align 32 [[ARG_RSRC]], i32 [[ARG_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[V_INT_SLICE_0]], ptr addrspace(8) align 32 [[ARG_RSRC]], i32 [[ARG_OFF]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[ARG_PART_4:%.*]] = add i32 [[ARG_OFF]], 16
; CHECK-NEXT: [[V_INT_SLICE_4:%.*]] = extractelement <5 x i32> [[V_INT_LEGAL]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[V_INT_SLICE_4]], ptr addrspace(8) align 16 [[ARG_RSRC]], i32 [[ARG_PART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[V_INT_SLICE_4]], ptr addrspace(8) align 16 [[ARG_RSRC]], i32 [[ARG_PART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
%v = call ptr addrspace(7) @extern(ptr addrspace(7) %arg)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
index e471178394303..9a94a12060f2d 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
@@ -212,7 +212,7 @@ define i32 @load_null() {
define void @store_null() {
; CHECK-LABEL: define void @store_null() {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: ret void
;
store i32 0, ptr addrspace(7) null, align 4
@@ -230,7 +230,7 @@ define i32 @load_poison() {
define void @store_poison() {
; CHECK-LABEL: define void @store_poison() {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
store i32 0, ptr addrspace(7) poison, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
index 95d0ed9114731..a221056406da2 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
@@ -37,22 +37,22 @@ define i32 @load_i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_i32_align4(i32 %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_i32_align4(
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align4(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align4(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align4(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; BOTH_FLAGS-NEXT: ret void
;
@@ -104,30 +104,30 @@ define void @store_i32_align2(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <2 x i16>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align2(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align2(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <2 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align2(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -190,42 +190,42 @@ define void @store_i32_align1(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <4 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align1(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align1(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <4 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align1(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -264,22 +264,22 @@ define i64 @load_i64_align4(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_i64_align4(i64 %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_i64_align4(
; STRICT-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i64_align4(
; UNALIGNED_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i64_align4(
; RELAXED_OOB_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i64_align4(
; BOTH_FLAGS-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -366,66 +366,66 @@ define void @store_i64_align1(i64 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i64 [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i64_align1(
; UNALIGNED_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i64_align1(
; RELAXED_OOB_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i64 [[DATA]] to <8 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i64_align1(
; BOTH_FLAGS-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -473,29 +473,29 @@ define void @store_v2i32_align4(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v2i32_align4(
; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align4(
; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align4(
; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i32_align4(
; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -562,46 +562,46 @@ define void @store_v2i32_align2(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <4 x i16>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align2(
; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align2(
; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <4 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i32_align2(
; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -692,70 +692,70 @@ define void @store_v2i32_align1(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align1(
; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align1(
; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <8 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i32_align1(
; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -809,35 +809,35 @@ define void @store_v3i32_align4(<3 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v3i32_align4(
; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <3 x i32> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <3 x i32> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align4(
; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <3 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <3 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align4(
; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v3i32_align4(
; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -887,29 +887,29 @@ define void @store_v3i32_align8(<3 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v3i32_align8(
; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i32> [[DATA]], <3 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align8(
; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i32> [[DATA]], <3 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align8(
; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v3i32_align8(
; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -948,22 +948,22 @@ define <3 x i32> @load_v3i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_v3i32_align16(<3 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_v3i32_align16(
; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align16(
; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align16(
; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v3i32_align16(
; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1015,29 +1015,29 @@ define void @store_v4i32_align8(<4 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v4i32_align8(
; STRICT-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i32_align8(
; UNALIGNED_ONLY-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i32_align8(
; RELAXED_OOB_ONLY-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i32_align8(
; BOTH_FLAGS-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1101,37 +1101,37 @@ define void @store_v8i32_align16(<8 x i32> %data, ptr addrspace(8) inreg %buf, i
; STRICT-LABEL: define void @store_v8i32_align16(
; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align16(
; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align16(
; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v8i32_align16(
; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1211,49 +1211,49 @@ define void @store_v8i32_align8(<8 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v8i32_align8(
; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align8(
; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align8(
; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v8i32_align8(
; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1349,73 +1349,73 @@ define void @store_v8i32_align4(<8 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v8i32_align4(
; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i32> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i32> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i32> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 12
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i32> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i32> [[DATA]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 20
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i32> [[DATA]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i32> [[DATA]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 28
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i32> [[DATA]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align4(
; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i32> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 12
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i32> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i32> [[DATA]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 20
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i32> [[DATA]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i32> [[DATA]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 28
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i32> [[DATA]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align4(
; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v8i32_align4(
; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1495,49 +1495,49 @@ define void @store_v10i32_align16(<10 x i32> %data, ptr addrspace(8) inreg %buf,
; STRICT-LABEL: define void @store_v10i32_align16(
; STRICT-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; STRICT-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v10i32_align16(
; UNALIGNED_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v10i32_align16(
; RELAXED_OOB_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v10i32_align16(
; BOTH_FLAGS-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; BOTH_FLAGS-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1633,61 +1633,61 @@ define void @store_v10i32_align8(<10 x i32> %data, ptr addrspace(8) inreg %buf,
; STRICT-LABEL: define void @store_v10i32_align8(
; STRICT-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; STRICT-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v10i32_align8(
; UNALIGNED_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v10i32_align8(
; RELAXED_OOB_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v10i32_align8(
; BOTH_FLAGS-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; BOTH_FLAGS-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1738,30 +1738,30 @@ define void @store_i16_align1(i16 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i16 [[DATA]] to <2 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i16_align1(
; UNALIGNED_ONLY-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i16_align1(
; RELAXED_OOB_ONLY-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i16 [[DATA]] to <2 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i16_align1(
; BOTH_FLAGS-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1809,29 +1809,29 @@ define void @store_v2i16_align4(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v2i16_align4(
; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align4(
; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align4(
; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i16_align4(
; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1883,33 +1883,33 @@ define void @store_v2i16_align2(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v2i16_align2(
; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align2(
; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align2(
; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i16_align2(
; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1976,46 +1976,46 @@ define void @store_v2i16_align1(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i16> [[DATA]] to <4 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align1(
; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align1(
; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i16> [[DATA]] to <4 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i16_align1(
; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2065,31 +2065,31 @@ define void @store_v2i8_align2(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v2i8_align2(
; STRICT-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i8_align2(
; UNALIGNED_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i8_align2(
; RELAXED_OOB_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i8_align2(
; BOTH_FLAGS-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2142,34 +2142,34 @@ define void @store_v2i8_align1(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v2i8_align1(
; STRICT-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i8_align1(
; UNALIGNED_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i8_align1(
; RELAXED_OOB_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i8_align1(
; BOTH_FLAGS-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2231,43 +2231,43 @@ define void @store_v4i8_align4(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v4i8_align4(
; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align4(
; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align4(
; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i8_align4(
; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2333,47 +2333,47 @@ define void @store_v4i8_align2(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v4i8_align2(
; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align2(
; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align2(
; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to <2 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i8_align2(
; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2444,52 +2444,52 @@ define void @store_v4i8_align1(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v4i8_align1(
; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align1(
; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align1(
; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i8_align1(
; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2578,68 +2578,68 @@ define void @store_v16i4_align8(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align8(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align8(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align8(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2728,68 +2728,68 @@ define void @store_v16i4_align4(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align4(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align4(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align4(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2888,78 +2888,78 @@ define void @store_v16i4_align2(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align2(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align2(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <4 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align2(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3070,90 +3070,90 @@ define void @store_v16i4_align1(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align1(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align1(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align1(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3278,3 +3278,12 @@ define <8 x half> @load_v8f16_align2_not_oob(ptr addrspace(1) inreg %ptr, i32 %o
!llvm.module.flags = !{!0}
!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 BUFFER_OOB_MODE}
+;.
+; STRICT: [[META1]] = !{}
+;.
+; UNALIGNED_ONLY: [[META1]] = !{}
+;.
+; RELAXED_OOB_ONLY: [[META1]] = !{}
+;.
+; BOTH_FLAGS: [[META1]] = !{}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
index 5efed9e569f2f..03fef762d8818 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
@@ -23,7 +23,7 @@ define i8 @load_i8(ptr addrspace(8) inreg %buf) {
define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i8(
; CHECK-SAME: i8 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1:![0-9]+]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -45,7 +45,7 @@ define i16 @load_i16(ptr addrspace(8) inreg %buf) {
define void @store_i16(i16 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i16(
; CHECK-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -67,7 +67,7 @@ define i32 @load_i32(ptr addrspace(8) inreg %buf) {
define void @store_i32(i32 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i32(
; CHECK-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -89,7 +89,7 @@ define i64 @load_i64(ptr addrspace(8) inreg %buf) {
define void @store_i64(i64 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i64(
; CHECK-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -111,7 +111,7 @@ define i128 @load_i128(ptr addrspace(8) inreg %buf) {
define void @store_i128(i128 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i128(
; CHECK-SAME: i128 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i128(i128 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i128(i128 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -135,7 +135,7 @@ define void @store_v1i32(<1 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i32(
; CHECK-SAME: <1 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -157,7 +157,7 @@ define <2 x i32> @load_v2i32(ptr addrspace(8) inreg %buf) {
define void @store_v2i32(<2 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i32(
; CHECK-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -179,7 +179,7 @@ define <3 x i32> @load_v3i32(ptr addrspace(8) inreg %buf) {
define void @store_v3i32(<3 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i32(
; CHECK-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -201,7 +201,7 @@ define <4 x i32> @load_v4i32(ptr addrspace(8) inreg %buf) {
define void @store_v4i32(<4 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i32(
; CHECK-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -223,7 +223,7 @@ define <2 x i16> @load_v2i16(ptr addrspace(8) inreg %buf) {
define void @store_v2i16(<2 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i16(
; CHECK-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -245,7 +245,7 @@ define <4 x i16> @load_v4i16(ptr addrspace(8) inreg %buf) {
define void @store_v4i16(<4 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i16(
; CHECK-SAME: <4 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -267,7 +267,7 @@ define <8 x i16> @load_v8i16(ptr addrspace(8) inreg %buf) {
define void @store_v8i16(<8 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i16(
; CHECK-SAME: <8 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -289,7 +289,7 @@ define <2 x i64> @load_v2i64(ptr addrspace(8) inreg %buf) {
define void @store_v2i64(<2 x i64> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i64(
; CHECK-SAME: <2 x i64> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i64(<2 x i64> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i64(<2 x i64> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -311,7 +311,7 @@ define half @load_f16(ptr addrspace(8) inreg %buf) {
define void @store_f16(half %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_f16(
; CHECK-SAME: half [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -333,7 +333,7 @@ define bfloat @load_bf16(ptr addrspace(8) inreg %buf) {
define void @store_bf16(bfloat %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_bf16(
; CHECK-SAME: bfloat [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -355,7 +355,7 @@ define <2 x half> @load_v2f16(ptr addrspace(8) inreg %buf) {
define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f16(
; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -381,9 +381,9 @@ define void @store_v2f16_align2(<2 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f16_align2(
; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x half> [[DATA]], i64 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x half> [[DATA]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -405,7 +405,7 @@ define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {
define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4bf16(
; CHECK-SAME: <4 x bfloat> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -427,7 +427,7 @@ define <8 x half> @load_v8f16(ptr addrspace(8) inreg %buf) {
define void @store_v8f16(<8 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8f16(
; CHECK-SAME: <8 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8f16(<8 x half> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8f16(<8 x half> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -449,7 +449,7 @@ define float @load_f32(ptr addrspace(8) inreg %buf) {
define void @store_f32(float %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_f32(
; CHECK-SAME: float [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -471,7 +471,7 @@ define <2 x float> @load_v2f32(ptr addrspace(8) inreg %buf) {
define void @store_v2f32(<2 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f32(
; CHECK-SAME: <2 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -493,7 +493,7 @@ define <3 x float> @load_v3f32(ptr addrspace(8) inreg %buf) {
define void @store_v3f32(<3 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3f32(
; CHECK-SAME: <3 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -515,7 +515,7 @@ define <4 x float> @load_v4f32(ptr addrspace(8) inreg %buf) {
define void @store_v4f32(<4 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4f32(
; CHECK-SAME: <4 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -537,7 +537,7 @@ define ptr addrspace(0) @load_p0(ptr addrspace(8) inreg %buf) {
define void @store_p0(ptr addrspace(0) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p0(
; CHECK-SAME: ptr [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -559,7 +559,7 @@ define ptr addrspace(1) @load_p1(ptr addrspace(8) inreg %buf) {
define void @store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p1(
; CHECK-SAME: ptr addrspace(1) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -581,7 +581,7 @@ define ptr addrspace(2) @load_p2(ptr addrspace(8) inreg %buf) {
define void @store_p2(ptr addrspace(2) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p2(
; CHECK-SAME: ptr addrspace(2) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p2(ptr addrspace(2) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p2(ptr addrspace(2) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -603,7 +603,7 @@ define ptr addrspace(3) @load_p3(ptr addrspace(8) inreg %buf) {
define void @store_p3(ptr addrspace(3) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p3(
; CHECK-SAME: ptr addrspace(3) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p3(ptr addrspace(3) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p3(ptr addrspace(3) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -625,7 +625,7 @@ define ptr addrspace(4) @load_p4(ptr addrspace(8) inreg %buf) {
define void @store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p4(
; CHECK-SAME: ptr addrspace(4) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -647,7 +647,7 @@ define ptr addrspace(5) @load_p5(ptr addrspace(8) inreg %buf) {
define void @store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p5(
; CHECK-SAME: ptr addrspace(5) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -669,7 +669,7 @@ define ptr addrspace(6) @load_p6(ptr addrspace(8) inreg %buf) {
define void @store_p6(ptr addrspace(6) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p6(
; CHECK-SAME: ptr addrspace(6) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p6(ptr addrspace(6) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p6(ptr addrspace(6) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -691,7 +691,7 @@ define ptr addrspace(8) @load_p8(ptr addrspace(8) inreg %buf) {
define void @store_p8(ptr addrspace(8) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p8(
; CHECK-SAME: ptr addrspace(8) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p8(ptr addrspace(8) [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p8(ptr addrspace(8) [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -713,7 +713,7 @@ define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(8) inreg %buf) {
define void @store_v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2p1(
; CHECK-SAME: <2 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -735,7 +735,7 @@ define <2 x ptr addrspace(5)> @load_v2p5(ptr addrspace(8) inreg %buf) {
define void @store_v2p5(<2 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2p5(
; CHECK-SAME: <2 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p5(<2 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p5(<2 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -757,7 +757,7 @@ define <3 x ptr addrspace(5)> @load_v3p5(ptr addrspace(8) inreg %buf) {
define void @store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3p5(
; CHECK-SAME: <3 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -779,7 +779,7 @@ define <4 x ptr addrspace(5)> @load_v4p5(ptr addrspace(8) inreg %buf) {
define void @store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4p5(
; CHECK-SAME: <4 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -805,7 +805,7 @@ define void @store_v6f16(<6 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6f16(
; CHECK-SAME: <6 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <6 x half> [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -834,9 +834,9 @@ define void @store_v5f32(<5 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5f32(
; CHECK-SAME: <5 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x float> [[DATA]], <5 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x float> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -864,9 +864,9 @@ define void @store_v6f32(<6 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6f32(
; CHECK-SAME: <6 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -894,9 +894,9 @@ define void @store_v7f32(<7 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v7f32(
; CHECK-SAME: <7 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <3 x i32> <i32 4, i32 5, i32 6>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -924,9 +924,9 @@ define void @store_v8f32(<8 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8f32(
; CHECK-SAME: <8 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -957,11 +957,11 @@ define void @store_v10f32(<10 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v10f32(
; CHECK-SAME: <10 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <2 x i32> <i32 8, i32 9>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_8]], ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_8]], ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -989,9 +989,9 @@ define void @store_v6i32(<6 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i32(
; CHECK-SAME: <6 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1019,9 +1019,9 @@ define void @store_v4p1(<4 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %bu
; CHECK-LABEL: define void @store_v4p1(
; CHECK-SAME: <4 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_2]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_2]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1047,7 +1047,7 @@ define void @store_v1i16(<1 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i16(
; CHECK-SAME: <1 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <1 x i16> [[DATA]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_STORABLE]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_STORABLE]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1074,9 +1074,9 @@ define void @store_v3i16(<3 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i16(
; CHECK-SAME: <3 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1103,9 +1103,9 @@ define void @store_v5i16(<5 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5i16(
; CHECK-SAME: <5 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i16> [[DATA]], <5 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i16> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1129,7 +1129,7 @@ define void @store_v6i16(<6 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i16(
; CHECK-SAME: <6 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <6 x i16> [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1158,9 +1158,9 @@ define void @store_v7i16(<7 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <7 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <7 x i16> [[DATA]], <7 x i16> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <6 x i16> [[DATA_SLICE_0]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <7 x i16> [[DATA]], i64 6
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1187,9 +1187,9 @@ define void @store_v9i16(<9 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v9i16(
; CHECK-SAME: <9 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <9 x i16> [[DATA]], <9 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_8:%.*]] = extractelement <9 x i16> [[DATA]], i64 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1217,7 +1217,7 @@ define void @store_v1i8(<1 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i8(
; CHECK-SAME: <1 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <1 x i8> [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1241,7 +1241,7 @@ define void @store_v2i8(<2 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i8(
; CHECK-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1270,9 +1270,9 @@ define void @store_v3i8(<3 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <3 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i8> [[DATA]], <3 x i8> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_0]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i8> [[DATA]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1296,7 +1296,7 @@ define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i8(
; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1325,9 +1325,9 @@ define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <5 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1356,9 +1356,9 @@ define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1391,12 +1391,12 @@ define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <7 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[DATA_SLICE_4_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_4]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <7 x i8> [[DATA]], i64 6
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1420,7 +1420,7 @@ define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i8(
; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <8 x i8> [[DATA]] to <2 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1458,21 +1458,21 @@ define void @store_v8i8_align1(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i8_align1(
; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA]], i64 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA]], i64 5
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA]], i64 6
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA]], i64 7
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1496,7 +1496,7 @@ define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v12i8(
; CHECK-SAME: <12 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <12 x i8> [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1520,7 +1520,7 @@ define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v16i8(
; CHECK-SAME: <16 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i8> [[DATA]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1550,9 +1550,9 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <32 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <32 x i8> [[DATA]] to <8 x i32>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1582,7 +1582,7 @@ define void @store_a1i32([1 x i32] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_ELEM_0:%.*]] = extractvalue [1 x i32] [[DATA]], 0
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <1 x i32> poison, i32 [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA_AS_VEC_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1612,7 +1612,7 @@ define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA_AS_VEC_0]], i32 [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1643,8 +1643,8 @@ define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x half> poison, half [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x half] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x half> [[DATA_AS_VEC_0]], half [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1674,7 +1674,7 @@ define void @store_a2p1([2 x ptr addrspace(1)] %data, ptr addrspace(8) inreg %bu
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x ptr addrspace(1)> poison, ptr addrspace(1) [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x ptr addrspace(1)] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x ptr addrspace(1)> [[DATA_AS_VEC_0]], ptr addrspace(1) [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_AS_VEC_1]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_AS_VEC_1]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1707,9 +1707,9 @@ define void @store_i40(i40 %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i40 [[DATA]] to <5 x i8>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA_LEGAL]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA_LEGAL]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1733,7 +1733,7 @@ define void @store_i96(i96 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i96(
; CHECK-SAME: i96 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i96 [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1762,9 +1762,9 @@ define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: i160 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i160 [[DATA]] to <5 x i32>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i32> [[DATA_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i32> [[DATA_LEGAL]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1794,9 +1794,9 @@ define void @store_i256(i256 %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: i256 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i256 [[DATA]] to <8 x i32>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1822,7 +1822,7 @@ define void @store_i7(i7 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i7(
; CHECK-SAME: i7 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_ZEXT:%.*]] = zext i7 [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1846,7 +1846,7 @@ define void @store_i4(i4 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i4(
; CHECK-SAME: i4 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_ZEXT:%.*]] = zext i4 [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1872,7 +1872,7 @@ define void @store_v2i4(<2 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i4(
; CHECK-SAME: <2 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i4> [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1896,7 +1896,7 @@ define void @store_v4i4(<4 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i4(
; CHECK-SAME: <4 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i4> [[DATA]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1920,7 +1920,7 @@ define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i4(
; CHECK-SAME: <8 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <8 x i4> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1948,7 +1948,7 @@ define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <2 x i6> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_AS_SCALAR:%.*]] = bitcast <2 x i6> [[DATA]] to i12
; CHECK-NEXT: [[DATA_ZEXT:%.*]] = zext i12 [[DATA_AS_SCALAR]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_ZEXT]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_ZEXT]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1981,9 +1981,9 @@ define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i32> [[DATA_ABI:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA:%.*]] = bitcast <6 x i32> [[DATA_ABI]] to <32 x i6>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%data = bitcast <6 x i32> %data.abi to <32 x i6>
@@ -2010,7 +2010,7 @@ define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @volatile_store_v4i8(
; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2039,9 +2039,9 @@ define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2079,13 +2079,13 @@ define void @store_a2a2i32([2 x [2 x i32]] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA0_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA0_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA0_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA0]], 1
; CHECK-NEXT: [[DATA0_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA0_AS_VEC_0]], i32 [[DATA0_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [2 x [2 x i32]] [[DATA]], 1
; CHECK-NEXT: [[DATA1_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA1]], 0
; CHECK-NEXT: [[DATA1_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA1_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA1_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA1]], 1
; CHECK-NEXT: [[DATA1_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA1_AS_VEC_0]], i32 [[DATA1_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2111,9 +2111,9 @@ define void @store_a2v2i32([2 x <2 x i32>] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2v2i32(
; CHECK-SAME: [2 x <2 x i32>] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA0:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2139,9 +2139,9 @@ define void @store_a2v3i32([2 x <3 x i32>] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2v3i32(
; CHECK-SAME: [2 x <3 x i32>] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA0:%.*]] = extractvalue [2 x <3 x i32>] [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [2 x <3 x i32>] [[DATA]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA1]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA1]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2165,7 +2165,7 @@ define void @store_sl_i32s({ i32 } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_i32s(
; CHECK-SAME: { i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0:%.*]] = extractvalue { i32 } [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2189,7 +2189,7 @@ define void @store_sl_sl_f32ss({ { float } } %data, ptr addrspace(8) inreg %buf)
; CHECK-LABEL: define void @store_sl_sl_f32ss(
; CHECK-SAME: { { float } } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0_0:%.*]] = extractvalue { { float } } [[DATA]], 0, 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_0_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_0_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2213,7 +2213,7 @@ define void @store_sl_v2i32s({ <2 x i32> } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_v2i32s(
; CHECK-SAME: { <2 x i32> } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0:%.*]] = extractvalue { <2 x i32> } [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2239,9 +2239,9 @@ define void @store_sl_i64i32s({ i64, i32 } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_i64i32s(
; CHECK-SAME: { i64, i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0:%.*]] = extractvalue { i64, i32 } [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_1:%.*]] = extractvalue { i64, i32 } [[DATA]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2276,16 +2276,16 @@ define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: [4 x i7] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA0:%.*]] = extractvalue [4 x i7] [[DATA]], 0
; CHECK-NEXT: [[DATA0_ZEXT:%.*]] = zext i7 [[DATA0]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA0_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA0_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [4 x i7] [[DATA]], 1
; CHECK-NEXT: [[DATA1_ZEXT:%.*]] = zext i7 [[DATA1]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA1_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA1_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA2:%.*]] = extractvalue [4 x i7] [[DATA]], 2
; CHECK-NEXT: [[DATA2_ZEXT:%.*]] = zext i7 [[DATA2]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA2_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA2_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA3:%.*]] = extractvalue [4 x i7] [[DATA]], 3
; CHECK-NEXT: [[DATA3_ZEXT:%.*]] = zext i7 [[DATA3]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA3_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA3_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2309,7 +2309,7 @@ define <vscale x 2 x i32> @load_nxv2i32(ptr addrspace(8) inreg %buf) {
define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_nxv2i32(
; CHECK-SAME: <vscale x 2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.nxv2i32(<vscale x 2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.nxv2i32(<vscale x 2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2319,3 +2319,6 @@ define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf
!llvm.module.flags = !{!0}
!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 1}
+;.
+; CHECK: [[META1]] = !{}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
index ddccc4c4f59cd..167036eb7d725 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
@@ -88,51 +88,51 @@ define void @memcpy_known(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg %d
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -161,13 +161,13 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -181,13 +181,13 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 false)
@@ -202,7 +202,7 @@ define void @memcpy_known_byte(ptr addrspace(7) inreg %src, ptr addrspace(7) inr
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 1, i1 false)
@@ -217,19 +217,19 @@ define void @memcpy_known_tail(ptr addrspace(7) inreg %src, ptr addrspace(7) inr
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 8
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0)
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i32 [[SRC_OFF]], 14
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0)
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[DST_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 15, i1 false)
@@ -441,133 +441,133 @@ define void @memcpy_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(7) inre
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP3]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP3]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP3]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP3]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP3]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP3]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP3]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP3]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP3]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP3]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP3]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP3]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP3]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP3]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP3]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP3]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP3]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP3]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP3]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP3]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP3]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP3]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP3]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP3]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP3]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP3]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP3]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP3]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP3]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP3]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP3]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP3]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP3]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP3]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP3]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP3]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP3]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP3]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP3]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP3]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP3]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP3]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP3]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP3]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP3]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -596,13 +596,13 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -616,13 +616,13 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 true)
@@ -655,13 +655,13 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP6]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP6]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP6]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP7]] = add i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -674,7 +674,7 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[SRC_OFF]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0)
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[DST_OFF]], [[TMP9]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP13]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_LOOP_EXPANSION]]
@@ -697,52 +697,52 @@ define void @memcpy_known_p1_to_p7(ptr addrspace(1) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 16
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1034,51 +1034,51 @@ define void @memcpy.inline_known(ptr addrspace(7) inreg %src, ptr addrspace(7) i
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1107,13 +1107,13 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -1127,13 +1127,13 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 false)
@@ -1148,7 +1148,7 @@ define void @memcpy.inline_known_byte(ptr addrspace(7) inreg %src, ptr addrspace
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 1, i1 false)
@@ -1163,19 +1163,19 @@ define void @memcpy.inline_known_tail(ptr addrspace(7) inreg %src, ptr addrspace
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 8
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0)
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i32 [[SRC_OFF]], 14
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0)
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[DST_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 15, i1 false)
@@ -1387,133 +1387,133 @@ define void @memcpy.inline_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP3]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP3]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP3]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP3]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP3]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP3]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP3]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP3]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP3]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP3]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP3]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP3]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP3]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP3]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP3]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP3]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP3]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP3]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP3]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP3]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP3]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP3]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP3]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP3]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP3]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP3]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP3]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP3]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP3]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP3]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP3]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP3]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP3]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP3]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP3]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP3]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP3]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP3]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP3]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP3]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP3]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP3]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP3]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP3]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP3]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1542,13 +1542,13 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -1562,13 +1562,13 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 true)
@@ -1601,13 +1601,13 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP6]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP6]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP6]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP7]] = add i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -1620,7 +1620,7 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[SRC_OFF]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0)
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[DST_OFF]], [[TMP9]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP13]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_LOOP_EXPANSION]]
@@ -1643,52 +1643,52 @@ define void @memcpy.inline_known_p1_to_p7(ptr addrspace(1) inreg %src, ptr addrs
; CHECK-NEXT: [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 16
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1910,37 +1910,37 @@ define void @memset_known(ptr addrspace(7) inreg %ptr) {
; CHECK: [[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[TMP1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -1956,21 +1956,21 @@ define void @memset_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 false)
@@ -1982,7 +1982,7 @@ define void @memset_known_byte(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 1, i1 false)
@@ -1994,13 +1994,13 @@ define void @memset_known_tail(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[PTR_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 15, i1 false)
@@ -2017,133 +2017,133 @@ define void @memset_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTC]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP2]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP2]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP2]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP2]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP2]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP2]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP2]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP2]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP2]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP2]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP2]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP2]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP2]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP2]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP2]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP2]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP2]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP2]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP2]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP2]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP2]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP2]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP2]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP2]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP2]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP2]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP2]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP2]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP2]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP2]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP2]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP2]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP2]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP2]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP2]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP2]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP2]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP2]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP2]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP2]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP2]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP2]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP2]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP2]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP2]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -2159,21 +2159,21 @@ define void @memset_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 true)
@@ -2192,13 +2192,13 @@ define void @memset_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %length) {
; CHECK: [[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[TMP2]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i32 [[TMP2]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -2209,7 +2209,7 @@ define void @memset_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %length) {
; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]] ], [ [[TMP10:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP4]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[PTR_OFF]], [[TMP8]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMSET_POST_LOOP_EXPANSION]]
@@ -2234,37 +2234,37 @@ define void @memset.inline_known(ptr addrspace(7) inreg %ptr) {
; CHECK: [[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[TMP1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -2280,21 +2280,21 @@ define void @memset.inline_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 false)
@@ -2306,7 +2306,7 @@ define void @memset.inline_known_byte(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 1, i1 false)
@@ -2318,13 +2318,13 @@ define void @memset.inline_known_tail(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[PTR_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 15, i1 false)
@@ -2341,133 +2341,133 @@ define void @memset.inline_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTC]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP2]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP2]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP2]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP2]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP2]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP2]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP2]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP2]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP2]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP2]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP2]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP2]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP2]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP2]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP2]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP2]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP2]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP2]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP2]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP2]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP2]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP2]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP2]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP2]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP2]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP2]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP2]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP2]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP2]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP2]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP2]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP2]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP2]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP2]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP2]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP2]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP2]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP2]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP2]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP2]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP2]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP2]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP2]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP2]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP2]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -2483,21 +2483,21 @@ define void @memset.inline_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 true)
@@ -2516,13 +2516,13 @@ define void @memset.inline_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %lengt
; CHECK: [[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[TMP2]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i32 [[TMP2]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -2533,7 +2533,7 @@ define void @memset.inline_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %lengt
; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]] ], [ [[TMP10:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP4]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[PTR_OFF]], [[TMP8]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMSET_POST_LOOP_EXPANSION]]
@@ -2559,37 +2559,37 @@ define void @memset_pattern_known(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[LOADSTORELOOP]] ]
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP5]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP5]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP5]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP5]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP5]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP5]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP5]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP5]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP5]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP5]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP5]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP5]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP5]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP5]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP5]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 64
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP3]], label %[[LOADSTORELOOP]], label %[[SPLIT:.*]]
@@ -2610,7 +2610,7 @@ define void @memset_pattern_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[DOTIDX:%.*]] = mul nsw i32 [[TMP1]], 4
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 32
; CHECK-NEXT: br i1 [[TMP4]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]], label %[[SPLIT:.*]]
@@ -2632,133 +2632,133 @@ define void @memset_pattern_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[DOTC]], 256
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP8]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP8]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP8]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP8]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP8]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP8]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP8]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP8]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP8]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP8]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP8]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP8]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP8]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP8]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP8]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP8]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP8]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP8]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP8]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP8]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP8]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP8]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP8]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP8]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP8]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP8]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP8]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP8]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP8]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP8]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP8]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP8]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP8]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP8]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP8]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP8]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP8]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP8]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP8]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP8]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP8]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP8]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP8]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP8]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP8]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP8]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP8]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP8]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP8]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP8]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP8]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP8]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP8]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP8]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP8]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP8]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP8]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP8]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP8]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP8]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP8]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP8]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP8]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i64 [[TMP1]], 64
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP5]], 8192
; CHECK-NEXT: br i1 [[TMP3]], label %[[LOADSTORELOOP]], label %[[SPLIT:.*]]
@@ -2779,7 +2779,7 @@ define void @memset_pattern_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[DOTIDX:%.*]] = mul nsw i32 [[TMP1]], 4
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 32
; CHECK-NEXT: br i1 [[TMP4]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]], label %[[SPLIT:.*]]
@@ -2803,13 +2803,13 @@ define void @memset_pattern_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %leng
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[LOADSTORELOOP]] ]
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i32 [[LOOP_INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[LOADSTORELOOP]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_COND]]
@@ -2821,7 +2821,7 @@ define void @memset_pattern_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %leng
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP8]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[DOTIDX:%.*]] = mul nsw i32 [[TMP2]], 4
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]], label %[[SPLIT]]
@@ -2849,3 +2849,6 @@ define void @llvm_amdgcn_load_to_lds(ptr addrspace(7) inreg %p, ptr addrspace(3)
call void @llvm.amdgcn.load.to.lds.p7(ptr addrspace(7) %q, ptr addrspace(3) %l, i32 4, i32 16, i32 0)
ret void
}
+;.
+; CHECK: [[META0]] = !{}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
index 74735529d723d..de955eebd1518 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
@@ -16,10 +16,10 @@ define void @loads(ptr addrspace(8) %buf) {
; CHECK-NEXT: [[VOLATILE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
; CHECK-NEXT: [[VOLATILE_NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648), !nontemporal [[META0]]
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2:![0-9]+]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
-; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
+; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3:![0-9]+]])
; CHECK-NEXT: fence acquire
; CHECK-NEXT: ret void
;
@@ -47,17 +47,17 @@ define void @loads(ptr addrspace(8) %buf) {
define void @stores(ptr addrspace(8) %buf, float %f, <4 x float> %f4) {
; CHECK-LABEL: define void @stores
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], <4 x float> [[F4:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !nontemporal [[META0]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648), !nontemporal [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]]), !nontemporal [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META1]]), !nontemporal [[META0]]
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence release
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3]])
; CHECK-NEXT: ret void
;
%base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -82,46 +82,46 @@ define void @atomicrmw(ptr addrspace(8) %buf, float %f, i32 %i) {
; CHECK-LABEL: define void @atomicrmw
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], i32 [[I:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[XCHG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.swap.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[XCHG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.swap.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[ADD:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[ADD:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[SUB:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[SUB:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[AND:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.and.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[AND:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.and.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[OR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.or.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[OR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.or.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[XOR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.xor.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[XOR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.xor.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[MIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[MIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[MAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[MAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[FADD:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[FADD:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[FMAX:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[FMAX:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[FMIN:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[FMIN:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: ret void
;
@@ -154,7 +154,7 @@ define {i32, i1} @cmpxchg(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
; CHECK-LABEL: define { i32, i1 } @cmpxchg
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]]
@@ -172,7 +172,7 @@ define {i32, i1} @cmpxchg_weak(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
; CHECK-LABEL: define { i32, i1 } @cmpxchg_weak
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
index 21d0caa40ea3e..76fe2f662b393 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -56,10 +56,10 @@ define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace
; CHECK-NEXT: #dbg_value(ptr addrspace(8) [[AUX_PTR_2_PTR_RSRC]], [[META21]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG34]])
; CHECK-NEXT: [[BUF_PTR_4_LEGAL:%.*]] = bitcast i160 [[BUF_PTR_4]] to <5 x i32>, !dbg [[DBG35:![0-9]+]]
; CHECK-NEXT: [[BUF_PTR_4_SLICE_0:%.*]] = shufflevector <5 x i32> [[BUF_PTR_4_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, !dbg [[DBG35]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0), !dbg [[DBG35]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0, metadata [[META8:![0-9]+]]), !dbg [[DBG35]]
; CHECK-NEXT: [[AUX_PTR_2_PTR_PART_4:%.*]] = add nuw i32 [[AUX_PTR_2_PTR_OFF]], 16, !dbg [[DBG35]]
; CHECK-NEXT: [[BUF_PTR_4_SLICE_4:%.*]] = extractelement <5 x i32> [[BUF_PTR_4_LEGAL]], i64 4, !dbg [[DBG35]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0), !dbg [[DBG35]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0, metadata [[META8]]), !dbg [[DBG35]]
; CHECK-NEXT: ret float [[RET]], !dbg [[DBG36:![0-9]+]]
;
%buf.ptr.var = alloca ptr addrspace(7), align 32, addrspace(5), !dbg !20
diff --git a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
index 0113437240236..f71844626cc91 100644
--- a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
+++ b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
; FNATTRS-NEXT: [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[P]], i16 0, i64 4, i32 822243328)
; FNATTRS-NEXT: [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[Q]], i16 0, i64 4, i32 822243328)
; FNATTRS-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[P_RSRC]], i32 0, i32 0, i32 0)
-; FNATTRS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) [[Q_RSRC]], i32 0, i32 0, i32 0)
+; FNATTRS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; FNATTRS-NEXT: ret void
;
; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
@@ -18,7 +18,7 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
; ATTRIBUTOR-NEXT: [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[P]], i16 0, i64 4, i32 822243328) #[[ATTR4:[0-9]+]]
; ATTRIBUTOR-NEXT: [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[Q]], i16 0, i64 4, i32 822243328) #[[ATTR4]]
; ATTRIBUTOR-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) readonly captures(none) [[P_RSRC]], i32 0, i32 0, i32 0) #[[ATTR5:[0-9]+]]
-; ATTRIBUTOR-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0) #[[ATTR6:[0-9]+]]
+; ATTRIBUTOR-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]]) #[[ATTR6:[0-9]+]]
; ATTRIBUTOR-NEXT: ret void
;
%p.rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 0, i64 4, i32 822243328)
diff --git a/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll b/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
index 51e2fb003d41b..65e8d85ca374a 100644
--- a/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
+++ b/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
@@ -13,7 +13,7 @@ define void @hoistable_noalias(ptr addrspace(8) noalias %p, ptr addrspace(8) noa
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -47,7 +47,7 @@ define void @not_hoistable_may_alias(ptr addrspace(8) %p, ptr addrspace(8) %q, i
; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[NOT_HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -75,13 +75,13 @@ define void @hoistable_alias_scope(ptr addrspace(8) %p, ptr addrspace(8) %q, i32
; CHECK-LABEL: define void @hoistable_alias_scope
; CHECK-SAME: (ptr addrspace(8) [[P:%.*]], ptr addrspace(8) [[Q:%.*]], i32 [[BOUND:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META1:![0-9]+]], !noalias [[META4:![0-9]+]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -129,7 +129,7 @@ define void @not_hoistable_buffer_construction(ptr addrspace(1) noalias %p.globa
; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[NOT_HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -173,7 +173,7 @@ define void @hoistable_buffer_construction_intrinsic(ptr addrspace(1) noalias %p
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -212,13 +212,13 @@ define void @hoistable_buffer_construction_alias_scope(ptr addrspace(1) %p.globa
; CHECK-NEXT: [[Q_EXT:%.*]] = zext i48 [[Q_TRUNC]] to i128
; CHECK-NEXT: [[P:%.*]] = inttoptr i128 [[P_EXT]] to ptr addrspace(8)
; CHECK-NEXT: [[Q:%.*]] = inttoptr i128 [[Q_EXT]] to ptr addrspace(8)
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META1]], !noalias [[META4]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
>From 8ee783c40a6dce6f181b31333c2659d88837f4e0 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 6 Aug 2026 11:21:47 +0200
Subject: [PATCH 4/6] clang
---
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 24 ++++++++++---
.../CodeGenOpenCL/builtins-amdgcn-gfx13.cl | 8 ++---
.../builtins-amdgcn-raw-buffer-atomic-add.cl | 9 +++--
.../builtins-amdgcn-raw-buffer-atomic-max.cl | 7 ++--
.../builtins-amdgcn-raw-buffer-store.cl | 36 +++++++++----------
5 files changed, 52 insertions(+), 32 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 29199f1726c1a..fa90a335d8cc3 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -31,6 +31,20 @@ using namespace llvm;
namespace {
+// Emits a raw.ptr.buffer.{store,atomic.*} intrinsic call with an appended
+// empty syncscope metadata arg, since the builtin form carries no scope info.
+static Value *emitAMDGPURawPtrBufferBuiltinWithScope(CodeGenFunction &CGF,
+ const CallExpr *E,
+ unsigned IntrinsicID) {
+ llvm::SmallVector<Value *, 6> Args;
+ for (unsigned I = 0; I < 5; ++I)
+ Args.push_back(CGF.EmitScalarExpr(E->getArg(I)));
+ Args.push_back(MetadataAsValue::get(CGF.getLLVMContext(),
+ MDNode::get(CGF.getLLVMContext(), {})));
+ Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Args[0]->getType());
+ return CGF.Builder.CreateCall(F, Args);
+}
+
static Value *emitAMDGPUSBufferLoadBuiltin(CodeGenFunction &CGF,
const CallExpr *E) {
llvm::Type *RetTy = CGF.ConvertType(E->getType());
@@ -2090,7 +2104,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_store);
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f32:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f16:
@@ -2155,19 +2169,19 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
EmitScalarExpr(E->getArg(4))});
}
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_add_i32:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_add);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f64:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax);
case AMDGPU::BI__builtin_amdgcn_s_buffer_load_i32:
case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2i32:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl
index f5d341b31991c..f9d7a3d3b16ac 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl
@@ -68,28 +68,28 @@ void test_cvt_scalef32_pk32_fp6_f32(global uint6 *out, float32 srcf32, float src
// CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[TMP0]], ptr addrspace(8) [[TMP1]], i32 [[TMP2]], i32 [[TMP3]], i32 0)
+// CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[TMP0]], ptr addrspace(8) [[TMP1]], i32 [[TMP2]], i32 [[TMP3]], i32 0, metadata [[META3:![0-9]+]])
// CHECK-NEXT: [[TMP5:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[FOUT_ADDR]], align 8
// CHECK-NEXT: store float [[TMP4]], ptr addrspace(1) [[TMP5]], align 4
// CHECK-NEXT: [[TMP6:%.*]] = load float, ptr addrspace(5) [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP7:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[TMP6]], ptr addrspace(8) [[TMP7]], i32 [[TMP8]], i32 [[TMP9]], i32 0)
+// CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[TMP6]], ptr addrspace(8) [[TMP7]], i32 [[TMP8]], i32 [[TMP9]], i32 0, metadata [[META3]])
// CHECK-NEXT: [[TMP11:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[FOUT_ADDR]], align 8
// CHECK-NEXT: store float [[TMP10]], ptr addrspace(1) [[TMP11]], align 4
// CHECK-NEXT: [[TMP12:%.*]] = load double, ptr addrspace(5) [[D_ADDR]], align 8
// CHECK-NEXT: [[TMP13:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double [[TMP12]], ptr addrspace(8) [[TMP13]], i32 [[TMP14]], i32 [[TMP15]], i32 0)
+// CHECK-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double [[TMP12]], ptr addrspace(8) [[TMP13]], i32 [[TMP14]], i32 [[TMP15]], i32 0, metadata [[META3]])
// CHECK-NEXT: [[TMP17:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[DOUT_ADDR]], align 8
// CHECK-NEXT: store double [[TMP16]], ptr addrspace(1) [[TMP17]], align 8
// CHECK-NEXT: [[TMP18:%.*]] = load double, ptr addrspace(5) [[D_ADDR]], align 8
// CHECK-NEXT: [[TMP19:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[TMP18]], ptr addrspace(8) [[TMP19]], i32 [[TMP20]], i32 [[TMP21]], i32 0)
+// CHECK-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[TMP18]], ptr addrspace(8) [[TMP19]], i32 [[TMP20]], i32 [[TMP21]], i32 0, metadata [[META3]])
// CHECK-NEXT: [[TMP23:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[DOUT_ADDR]], align 8
// CHECK-NEXT: store double [[TMP22]], ptr addrspace(1) [[TMP23]], align 8
// CHECK-NEXT: ret void
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl
index 95abeb92e59e7..079d36ccfef40 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl
@@ -10,7 +10,7 @@ typedef half __attribute__((ext_vector_type(2))) float16x2_t;
// CHECK-LABEL: define dso_local i32 @test_atomic_add_i32(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], i32 noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret i32 [[TMP0]]
//
int test_atomic_add_i32(__amdgpu_buffer_rsrc_t rsrc, int x, int offset, int soffset) {
@@ -20,7 +20,7 @@ int test_atomic_add_i32(__amdgpu_buffer_rsrc_t rsrc, int x, int offset, int soff
// CHECK-LABEL: define dso_local float @test_atomic_fadd_f32(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], float noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret float [[TMP0]]
//
float test_atomic_fadd_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int soffset) {
@@ -30,9 +30,12 @@ float test_atomic_fadd_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int
// CHECK-LABEL: define dso_local <2 x half> @test_atomic_fadd_v2f16(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], <2 x half> noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <2 x half> [[TMP0]]
//
float16x2_t test_atomic_fadd_v2f16(__amdgpu_buffer_rsrc_t rsrc, float16x2_t x, int offset, int soffset) {
return __builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16(x, rsrc, offset, soffset, 0);
}
+//.
+// CHECK: [[META8]] = !{}
+//.
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl
index feb8682d522b1..21d2c627d5a7b 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl
@@ -8,7 +8,7 @@
// CHECK-LABEL: define dso_local float @test_atomic_fmax_f32(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], float noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret float [[TMP0]]
//
float test_atomic_fmax_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int soffset) {
@@ -18,9 +18,12 @@ float test_atomic_fmax_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int
// CHECK-LABEL: define dso_local double @test_atomic_fmax_f64(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], double noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret double [[TMP0]]
//
double test_atomic_fmax_f64(__amdgpu_buffer_rsrc_t rsrc, double x, int offset, int soffset) {
return __builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64(x, rsrc, offset, soffset, 0);
}
+//.
+// CHECK: [[META8]] = !{}
+//.
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
index e2dfa50ee3a2d..c07126a1584b4 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
@@ -11,7 +11,7 @@ typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b8(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -20,7 +20,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8(u8 vdata, __amdgpu_buffer_rsrc_t rsrc,
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b16(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -29,7 +29,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16(u16 vdata, __amdgpu_buffer_rsrc_t rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b32(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -38,7 +38,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32(u32 vdata, __amdgpu_buffer_rsrc_t rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b64(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b64(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -47,7 +47,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64(v2u32 vdata, __amdgpu_buffer_rsrc_t rs
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b96(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b96(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -56,7 +56,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96(v3u32 vdata, __amdgpu_buffer_rsrc_t rs
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b128(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b128(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -65,7 +65,7 @@ void test_amdgcn_raw_ptr_buffer_store_b128(v4u32 vdata, __amdgpu_buffer_rsrc_t r
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -74,7 +74,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(u8 vdata, __amdgpu_buf
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -83,7 +83,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(u16 vdata, __amdgpu_b
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -92,7 +92,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(u32 vdata, __amdgpu_b
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -101,7 +101,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(v2u32 vdata, __amdgpu
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -110,7 +110,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(v3u32 vdata, __amdgpu
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -119,7 +119,7 @@ void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(v4u32 vdata, __amdgp
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -128,7 +128,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(u8 vdata, __amdgpu_bu
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -137,7 +137,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(u16 vdata, __amdgpu_
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -146,7 +146,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(u32 vdata, __amdgpu_
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -155,7 +155,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(v2u32 vdata, __amdgp
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -164,7 +164,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(v3u32 vdata, __amdgp
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b128_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b128_non_const_soffset(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
>From d4c01a26bfe6d8097821f1820cd1dfc8e54266bf Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 6 Aug 2026 13:24:54 +0200
Subject: [PATCH 5/6] mlir
---
mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td | 15 ++++----
.../AMDGPUToROCDL/AMDGPUToROCDL.cpp | 9 +++++
.../AMDGPUToROCDL/amdgpu-to-rocdl.mlir | 28 +++++++--------
mlir/test/Dialect/LLVMIR/rocdl.mlir | 36 ++++++++++---------
mlir/test/Target/LLVMIR/rocdl.mlir | 36 ++++++++++---------
5 files changed, 72 insertions(+), 52 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index f1195e9c644d5..e67408a5e8c00 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -2075,9 +2075,10 @@ def ROCDL_RawPtrBufferStoreOp :
Arg<ROCDLBufferRsrc, "", [MemWrite]>:$rsrc,
I32:$offset,
I32:$soffset,
- ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux,
+ Arg<LLVM_MetadataType, "Sync scope for the access, or an empty metadata node if unspecified">:$syncscope);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $syncscope attr-dict `:` type($vdata)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2094,9 +2095,10 @@ def ROCDL_RawPtrBufferAtomicCmpSwap :
Arg<ROCDLBufferRsrc, "", [MemRead, MemWrite]>:$rsrc,
I32:$offset,
I32:$soffset,
- ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux,
+ Arg<LLVM_MetadataType, "Sync scope for the access, or an empty metadata node if unspecified">:$syncscope);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$src `,` $cmp `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($res)";
+ let assemblyFormat = "$src `,` $cmp `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $syncscope attr-dict `:` type($res)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2111,9 +2113,10 @@ class ROCDL_RawPtrBufferAtomicRet<string op> :
Arg<ROCDLBufferRsrc, "", [MemRead, MemWrite]>:$rsrc,
I32:$offset,
I32:$soffset,
- ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux,
+ Arg<LLVM_MetadataType, "Sync scope for the access, or an empty metadata node if unspecified">:$syncscope);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $syncscope attr-dict `:` type($vdata)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 90e15b1680446..97aa2ad6508b2 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -34,6 +34,7 @@
#include "llvm/Support/ErrorHandling.h"
#include <cstdint>
#include <optional>
+#include <type_traits>
namespace mlir {
#define GEN_PASS_DEF_CONVERTAMDGPUTOROCDLPASS
@@ -488,6 +489,14 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
args.push_back(sgprOffset);
+ if constexpr (!std::is_same_v<Intrinsic, ROCDL::RawPtrBufferLoadOp>) {
+ // No syncscope information is available at this level, so conservatively
+ // report an empty (unspecified) scope.
+ Value noScope = LLVM::MetadataAsValueOp::create(
+ rewriter, loc, LLVM::MDNodeAttr::get(rewriter.getContext(), {}));
+ args.push_back(noScope);
+ }
+
llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
llvmBufferValType);
typename Intrinsic::Properties properties;
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
index f6c0241a89484..73e7e2e568204 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
@@ -273,7 +273,7 @@ func.func @gpu_gcn_raw_buffer_store_scalar_i32(%value: i32, %buf: memref<i32>) {
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %{{.*}}, %[[flags]]
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[] : i32 -> memref<i32>
func.return
}
@@ -285,7 +285,7 @@ func.func @gpu_gcn_raw_buffer_store_i32(%value: i32, %buf: memref<64xi32>, %idx:
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : i32 -> memref<64xi32>, i32
func.return
}
@@ -293,7 +293,7 @@ func.func @gpu_gcn_raw_buffer_store_i32(%value: i32, %buf: memref<64xi32>, %idx:
// CHECK-LABEL: func @gpu_gcn_raw_buffer_store_1xf32
func.func @gpu_gcn_raw_buffer_store_1xf32(%value: vector<1xf32>, %buf: memref<64xf32>, %idx: i32) {
// CHECK: %[[cast:.*]] = llvm.bitcast %{{.*}} : vector<1xf32> to f32
- // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : vector<1xf32> -> memref<64xf32>, i32
func.return
}
@@ -301,7 +301,7 @@ func.func @gpu_gcn_raw_buffer_store_1xf32(%value: vector<1xf32>, %buf: memref<64
// CHECK-LABEL: func @gpu_gcn_raw_buffer_store_2xi8
func.func @gpu_gcn_raw_buffer_store_2xi8(%value: vector<2xi8>, %buf: memref<64xi8>, %idx: i32) {
// CHECK: %[[cast:.*]] = llvm.bitcast %{{.*}} : vector<2xi8> to i16
- // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : i16
+ // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i16
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : vector<2xi8> -> memref<64xi8>, i32
func.return
}
@@ -309,7 +309,7 @@ func.func @gpu_gcn_raw_buffer_store_2xi8(%value: vector<2xi8>, %buf: memref<64xi
// CHECK-LABEL: func @gpu_gcn_raw_buffer_store_16xi8
func.func @gpu_gcn_raw_buffer_store_16xi8(%value: vector<16xi8>, %buf: memref<64xi8>, %idx: i32) {
// CHECK: %[[cast:.*]] = llvm.bitcast %{{.*}} : vector<16xi8> to vector<4xi32>
- // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xi32>
+ // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<4xi32>
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : vector<16xi8> -> memref<64xi8>, i32
func.return
}
@@ -322,7 +322,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fadd_f32(%value: f32, %buf: memref<64xf32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fadd {boundsCheck = true} %value -> %buf[%idx] : f32 -> memref<64xf32>, i32
func.return %old : f32
@@ -335,7 +335,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fadd_v2f16(%value: vector<2xf16>, %buf: mem
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : vector<2xf16>
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<2xf16>
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fadd {boundsCheck = true} %value -> %buf[%idx] : vector<2xf16> -> memref<64xf16>, i32
func.return %old : vector<2xf16>
@@ -348,7 +348,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fadd_v2bf16(%value: vector<2xbf16>, %buf: m
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : vector<2xbf16>
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<2xbf16>
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fadd {boundsCheck = true} %value -> %buf[%idx] : vector<2xbf16> -> memref<64xbf16>, i32
func.return %old : vector<2xbf16>
@@ -361,7 +361,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fmax_f32(%value: f32, %buf: memref<64xf32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fmax {boundsCheck = true} %value -> %buf[%idx] : f32 -> memref<64xf32>, i32
func.return %old : f32
@@ -374,7 +374,7 @@ func.func @gpu_gcn_raw_buffer_atomic_smax_i32(%value: i32, %buf: memref<64xi32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_smax {boundsCheck = true} %value -> %buf[%idx] : i32 -> memref<64xi32>, i32
func.return %old : i32
@@ -387,7 +387,7 @@ func.func @gpu_gcn_raw_buffer_atomic_umin_i32(%value: i32, %buf: memref<64xi32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_umin {boundsCheck = true} %value -> %buf[%idx] : i32 -> memref<64xi32>, i32
func.return %old : i32
@@ -403,7 +403,7 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_f32(%src : f32, %cmp : f32, %buf : m
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcCast]], %[[cmpCast]], %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcCast]], %[[cmpCast]], %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: %[[dstCast:.*]] = llvm.bitcast %[[dst]] : i32 to f32
// CHECK: return %[[dstCast]]
%dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : f32 -> memref<64xf32>, i32
@@ -418,7 +418,7 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_i64(%src : i64, %cmp : i64, %buf : m
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[src]], %[[cmp]], %[[resource]], %{{.*}}, %{{.*}}, 0 : i64
+ // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[src]], %[[cmp]], %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i64
// CHECK: return %[[dst]]
%dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : i64 -> memref<64xi64>, i32
func.return %dst : i64
@@ -429,7 +429,7 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_i64(%src : i64, %cmp : i64, %buf : m
func.func @amdgpu_raw_buffer_atomic_cmpswap_v2f16(%src : vector<2xf16>, %cmp : vector<2xf16>, %buf : memref<64xf16>, %idx: i32) -> vector<2xf16> {
// CHECK-DAG: %[[srcBits:.+]] = llvm.bitcast %[[src]] : vector<2xf16> to i32
// CHECK-DAG: %[[cmpBits:.+]] = llvm.bitcast %[[cmp]] : vector<2xf16> to i32
- // CHECK: %[[dstBits:.+]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcBits]], %[[cmpBits]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[dstBits:.+]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcBits]], %[[cmpBits]], %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: %[[dst:.+]] = llvm.bitcast %[[dstBits]] : i32 to vector<2xf16>
// CHECK: return %[[dst]]
%dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : vector<2xf16> -> memref<64xf16>, i32
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 416f2edca3fbe..aac462ded97af 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -888,23 +888,25 @@ llvm.func @rocdl.raw.ptr.buffer.f32(%rsrc : !llvm.ptr<8>,
// CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
// CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<2xf32>
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<4xf32>
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
+
+ %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
%r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : f32
%r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xf32>
%r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xf32>
- rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0 : f32
- rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0 : vector<2xf32>
- rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, 0 : vector<4xf32>
+ rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
+ rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0, %scope : vector<2xf32>
+ rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, 0, %scope : vector<4xf32>
- %atomic_fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0 : f32
- %atomic_fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ %atomic_fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
+ %atomic_fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
llvm.return
}
@@ -945,13 +947,15 @@ llvm.func @rocdl.raw.ptr.buffer.i32(%rsrc : !llvm.ptr<8>,
%aux : i32, %vdata1 : i32,
%vdata2 : vector<2xi32>, %vdata4 : vector<4xi32>) {
// CHECK-LABEL: rocdl.raw.ptr.buffer.i32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.cmpswap %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.cmpswap %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
+
+ %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
- %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0 : i32
- %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0 : i32
- %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
+ %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
+ %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 2a001fb860c3d..13fb3a1948abe 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -1524,22 +1524,23 @@ llvm.func @rocdl.raw.ptr.buffer(%rsrc : !llvm.ptr<8>,
%vdata2 : vector<2xi32>,
%vdata4 : vector<4xi32>) {
%aux = llvm.mlir.constant(0 : i32) : i32
+ %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer
// CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- // CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- // CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- // CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
+ // CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
+ // CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
+ // CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
%r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : i32
%r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xi32>
%r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xi32>
- rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0 : i32
- rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0 : vector<2xi32>
- rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %soffset, 0 : vector<4xi32>
+ rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
+ rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0, %scope : vector<2xi32>
+ rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %soffset, 0, %scope : vector<4xi32>
llvm.return
}
@@ -1690,13 +1691,14 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.f32(%rsrc : !llvm.ptr<8>,
%offset : i32, %soffset : i32,
%vdata1 : f32) -> f32 {
%aux = llvm.mlir.constant(0 : i32) : i32
+ %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer.atomic.f32
- // CHECK: [[PTR_FADD:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- // CHECK: [[PTR_FMAX:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
+ // CHECK: [[PTR_FADD:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
+ // CHECK: [[PTR_FMAX:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: ret float [[PTR_FMAX]]
- %fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0 : f32
- %fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ %fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
+ %fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
llvm.return %fmax : f32
}
@@ -1705,13 +1707,14 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.i32(%rsrc : !llvm.ptr<8>,
%offset : i32, %soffset : i32,
%vdata1 : i32) -> i32 {
%aux = llvm.mlir.constant(0 : i32) : i32
+ %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer.atomic.i32
- // CHECK: [[PTR_SMAX:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- // CHECK: [[PTR_UMIN:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
+ // CHECK: [[PTR_SMAX:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
+ // CHECK: [[PTR_UMIN:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: ret i32 [[PTR_UMIN]]
- %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0 : i32
- %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
+ %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
llvm.return %umin : i32
}
@@ -1720,11 +1723,12 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.cmpswap(%rsrc : !llvm.ptr<8>,
%offset : i32, %soffset : i32,
%src : i32, %cmp : i32) -> i32 {
%aux = llvm.mlir.constant(0 : i32) : i32
+ %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer.atomic.cmpswap
- // CHECK: [[val:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %{{.*}}, i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
+ // CHECK: [[val:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %{{.*}}, i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: ret i32 [[val]]
- %val = rocdl.raw.ptr.buffer.atomic.cmpswap %src, %cmp, %rsrc, %offset, %soffset, 0 : i32
+ %val = rocdl.raw.ptr.buffer.atomic.cmpswap %src, %cmp, %rsrc, %offset, %soffset, 0, %scope : i32
llvm.return %val : i32
}
>From ebe5f8255e7e80c3ad2fdca8e0f58db34603fe04 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 10 Aug 2026 06:21:11 +0200
Subject: [PATCH 6/6] Address comments
---
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 69 +-
.../builtins-amdgcn-buffer-format.hip | 16 +-
.../builtins-amdgcn-raw-buffer-load-format.cl | 8 +-
.../builtins-amdgcn-raw-buffer-load.cl | 36 +-
...builtins-amdgcn-raw-buffer-store-format.cl | 8 +-
...iltins-amdgcn-struct-buffer-load-format.cl | 6 +-
...ltins-amdgcn-struct-buffer-store-format.cl | 6 +-
llvm/docs/AMDGPUUsage.rst | 30 +
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 61 +-
llvm/lib/IR/AutoUpgrade.cpp | 208 ++---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 6 +-
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 37 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 67 +-
....ll => amdgpu-buffer-atomicity-upgrade.ll} | 14 +-
.../atomic_optimizations_mul_one.ll | 16 +-
.../AMDGPU/GlobalISel/atomicrmw_fmax.ll | 60 +-
.../AMDGPU/GlobalISel/atomicrmw_fmin.ll | 60 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 787 ++++++++++++++----
.../buffer-fat-pointer-atomicrmw-fmax.ll | 603 +++++++++++---
.../buffer-fat-pointer-atomicrmw-fmin.ll | 603 +++++++++++---
.../buffer-fat-pointer-atomicrmw-usub_cond.ll | 32 +-
.../buffer-fat-pointer-atomicrmw-usub_sat.ll | 37 +-
.../lower-buffer-fat-pointers-constants.ll | 6 +-
...ointers-contents-legalization-alignment.ll | 710 ++++++++--------
...ffer-fat-pointers-contents-legalization.ll | 254 +++---
.../lower-buffer-fat-pointers-control-flow.ll | 18 +-
.../lower-buffer-fat-pointers-mem-transfer.ll | 566 ++++++-------
.../lower-buffer-fat-pointers-memops.ll | 38 +-
.../lower-buffer-fat-pointers-pointer-ops.ll | 2 +-
...fer-fat-pointers-unoptimized-debug-data.ll | 4 +-
.../AMDGPU/memory-legalizer-buffer-atomics.ll | 23 +
.../FunctionAttrs/make-buffer-rsrc.ll | 8 +-
...mdgcn-demanded-vector-elts-inseltpoison.ll | 240 +++---
.../AMDGPU/amdgcn-demanded-vector-elts.ll | 240 +++---
.../LICM/AMDGPU/buffer-rsrc-ptrs.ll | 26 +-
mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td | 17 +-
.../AMDGPUToROCDL/AMDGPUToROCDL.cpp | 10 +-
.../AMDGPUToROCDL/amdgpu-to-rocdl.mlir | 22 +-
mlir/test/Dialect/LLVMIR/rocdl.mlir | 32 +-
mlir/test/Target/LLVMIR/rocdl.mlir | 30 +-
40 files changed, 3140 insertions(+), 1876 deletions(-)
rename llvm/test/Bitcode/{amdgpu-buffer-atomic-scope-upgrade.ll => amdgpu-buffer-atomicity-upgrade.ll} (67%)
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 9cad91b88fbb0..6fb25d4bc5ed6 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -32,17 +32,18 @@ using namespace llvm;
namespace {
-// Emits a raw.ptr.buffer.{store,atomic.*} intrinsic call with an appended
-// empty syncscope metadata arg, since the builtin form carries no scope info.
-static Value *emitAMDGPURawPtrBufferBuiltinWithScope(CodeGenFunction &CGF,
- const CallExpr *E,
- unsigned IntrinsicID) {
- llvm::SmallVector<Value *, 6> Args;
- for (unsigned I = 0; I < 5; ++I)
+// No builtin can express atomicity, so the trailing atomicity metadata arg is
+// always the empty node. OverloadTy defaults to the first argument type.
+static Value *emitAMDGPUBufferBuiltin(CodeGenFunction &CGF, const CallExpr *E,
+ unsigned IntrinsicID,
+ llvm::Type *OverloadTy = nullptr) {
+ llvm::SmallVector<Value *, 7> Args;
+ for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
Args.push_back(CGF.EmitScalarExpr(E->getArg(I)));
Args.push_back(MetadataAsValue::get(CGF.getLLVMContext(),
MDNode::get(CGF.getLLVMContext(), {})));
- Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Args[0]->getType());
+ Function *F = CGF.CGM.getIntrinsic(
+ IntrinsicID, OverloadTy ? OverloadTy : Args[0]->getType());
return CGF.Builder.CreateCall(F, Args);
}
@@ -2129,11 +2130,11 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
- return emitAMDGPURawPtrBufferBuiltinWithScope(
- *this, E, Intrinsic::amdgcn_raw_ptr_buffer_store);
+ return emitAMDGPUBufferBuiltin(*this, E,
+ Intrinsic::amdgcn_raw_ptr_buffer_store);
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f32:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f16:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPUBufferBuiltin(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_store_format);
case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
@@ -2162,51 +2163,37 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/4);
break;
}
- Function *F =
- CGM.getIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_load, RetTy);
- return Builder.CreateCall(
- F, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)),
- EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3))});
+ return emitAMDGPUBufferBuiltin(
+ *this, E, Intrinsic::amdgcn_raw_ptr_buffer_load, RetTy);
}
case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f32:
- case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f16: {
- llvm::Type *RetTy = ConvertType(E->getType());
- Function *F =
- CGM.getIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_load_format, {RetTy});
-
- return Builder.CreateCall(
- F, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)),
- EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3))});
- }
+ case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f16:
+ return emitAMDGPUBufferBuiltin(*this, E,
+ Intrinsic::amdgcn_raw_ptr_buffer_load_format,
+ ConvertType(E->getType()));
case AMDGPU::BI__builtin_amdgcn_struct_buffer_store_format_v4f32:
case AMDGPU::BI__builtin_amdgcn_struct_buffer_store_format_v4f16:
- return emitBuiltinWithOneOverloadedType<6>(
+ return emitAMDGPUBufferBuiltin(
*this, E, Intrinsic::amdgcn_struct_ptr_buffer_store_format);
case AMDGPU::BI__builtin_amdgcn_struct_buffer_load_format_v4f32:
- case AMDGPU::BI__builtin_amdgcn_struct_buffer_load_format_v4f16: {
- llvm::Type *RetTy = ConvertType(E->getType());
- Function *F = CGM.getIntrinsic(
- Intrinsic::amdgcn_struct_ptr_buffer_load_format, {RetTy});
-
- return Builder.CreateCall(
- F, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)),
- EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3)),
- EmitScalarExpr(E->getArg(4))});
- }
+ case AMDGPU::BI__builtin_amdgcn_struct_buffer_load_format_v4f16:
+ return emitAMDGPUBufferBuiltin(
+ *this, E, Intrinsic::amdgcn_struct_ptr_buffer_load_format,
+ ConvertType(E->getType()));
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_add_i32:
- return emitAMDGPURawPtrBufferBuiltinWithScope(
- *this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_add);
+ return emitAMDGPUBufferBuiltin(*this, E,
+ Intrinsic::amdgcn_raw_ptr_buffer_atomic_add);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16:
- return emitAMDGPURawPtrBufferBuiltinWithScope(
+ return emitAMDGPUBufferBuiltin(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f64:
- return emitAMDGPURawPtrBufferBuiltinWithScope(
+ return emitAMDGPUBufferBuiltin(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64:
- return emitAMDGPURawPtrBufferBuiltinWithScope(
+ return emitAMDGPUBufferBuiltin(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax);
case AMDGPU::BI__builtin_amdgcn_s_buffer_load_i32:
case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2i32:
diff --git a/clang/test/CodeGenHIP/builtins-amdgcn-buffer-format.hip b/clang/test/CodeGenHIP/builtins-amdgcn-buffer-format.hip
index 772ea080cf9c5..3adb5de4becaf 100644
--- a/clang/test/CodeGenHIP/builtins-amdgcn-buffer-format.hip
+++ b/clang/test/CodeGenHIP/builtins-amdgcn-buffer-format.hip
@@ -9,7 +9,7 @@ typedef _Float16 v4f16 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @_Z33test_raw_buffer_load_format_v4f32u22__amdgpu_buffer_rsrc_tii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
__device__ v4f32 test_raw_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -18,7 +18,7 @@ __device__ v4f32 test_raw_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc,
// CHECK-LABEL: @_Z33test_raw_buffer_load_format_v4f16u22__amdgpu_buffer_rsrc_tii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x half> [[TMP0]]
//
__device__ v4f16 test_raw_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -27,7 +27,7 @@ __device__ v4f16 test_raw_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc,
// CHECK-LABEL: @_Z34test_raw_buffer_store_format_v4f32Dv4_fu22__amdgpu_buffer_rsrc_tii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
__device__ void test_raw_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -36,7 +36,7 @@ __device__ void test_raw_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_
// CHECK-LABEL: @_Z34test_raw_buffer_store_format_v4f16Dv4_DF16_u22__amdgpu_buffer_rsrc_tii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
__device__ void test_raw_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -45,7 +45,7 @@ __device__ void test_raw_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_
// CHECK-LABEL: @_Z36test_struct_buffer_load_format_v4f32u22__amdgpu_buffer_rsrc_tiii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
__device__ v4f32 test_struct_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc, int vindex, int offset, int soffset) {
@@ -54,7 +54,7 @@ __device__ v4f32 test_struct_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsr
// CHECK-LABEL: @_Z36test_struct_buffer_load_format_v4f16u22__amdgpu_buffer_rsrc_tiii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call contract <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x half> [[TMP0]]
//
__device__ v4f16 test_struct_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc, int vindex, int offset, int soffset) {
@@ -63,7 +63,7 @@ __device__ v4f16 test_struct_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsr
// CHECK-LABEL: @_Z37test_struct_buffer_store_format_v4f32Dv4_fu22__amdgpu_buffer_rsrc_tiii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
__device__ void test_struct_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc, int vindex, int offset, int soffset) {
@@ -72,7 +72,7 @@ __device__ void test_struct_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buff
// CHECK-LABEL: @_Z37test_struct_buffer_store_format_v4f16Dv4_DF16_u22__amdgpu_buffer_rsrc_tiii(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
__device__ void test_struct_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_rsrc_t rsrc, int vindex, int offset, int soffset) {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load-format.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load-format.cl
index 9f87769c2fd5b..4086b5ff2c549 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load-format.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load-format.cl
@@ -9,7 +9,7 @@ typedef half v4f16 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_raw_buffer_load_format_v4f32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
v4f32 test_raw_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc) {
@@ -18,7 +18,7 @@ v4f32 test_raw_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc) {
// CHECK-LABEL: @test_raw_buffer_load_format_v4f16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x half> [[TMP0]]
//
v4f16 test_raw_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc) {
@@ -27,7 +27,7 @@ v4f16 test_raw_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc) {
// CHECK-LABEL: @test_raw_buffer_load_format_v4f32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
v4f32 test_raw_buffer_load_format_v4f32_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset) {
@@ -36,7 +36,7 @@ v4f32 test_raw_buffer_load_format_v4f32_non_const_offset(__amdgpu_buffer_rsrc_t
// CHECK-LABEL: @test_raw_buffer_load_format_v4f32_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
v4f32 test_raw_buffer_load_format_v4f32_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int soffset) {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl
index cfb356c684f64..ea18a343f62d4 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-load.cl
@@ -11,7 +11,7 @@ typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b8(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret i8 [[TMP0]]
//
u8 test_amdgcn_raw_ptr_buffer_load_b8(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -20,7 +20,7 @@ u8 test_amdgcn_raw_ptr_buffer_load_b8(__amdgpu_buffer_rsrc_t rsrc, int offset, i
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret i16 [[TMP0]]
//
u16 test_amdgcn_raw_ptr_buffer_load_b16(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -29,7 +29,7 @@ u16 test_amdgcn_raw_ptr_buffer_load_b16(__amdgpu_buffer_rsrc_t rsrc, int offset,
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret i32 [[TMP0]]
//
u32 test_amdgcn_raw_ptr_buffer_load_b32(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -38,7 +38,7 @@ u32 test_amdgcn_raw_ptr_buffer_load_b32(__amdgpu_buffer_rsrc_t rsrc, int offset,
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b64(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
v2u32 test_amdgcn_raw_ptr_buffer_load_b64(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -47,7 +47,7 @@ v2u32 test_amdgcn_raw_ptr_buffer_load_b64(__amdgpu_buffer_rsrc_t rsrc, int offse
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b96(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <3 x i32> [[TMP0]]
//
v3u32 test_amdgcn_raw_ptr_buffer_load_b96(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -56,7 +56,7 @@ v3u32 test_amdgcn_raw_ptr_buffer_load_b96(__amdgpu_buffer_rsrc_t rsrc, int offse
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b128(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_raw_ptr_buffer_load_b128(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -65,7 +65,7 @@ v4u32 test_amdgcn_raw_ptr_buffer_load_b128(__amdgpu_buffer_rsrc_t rsrc, int offs
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b8_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret i8 [[TMP0]]
//
u8 test_amdgcn_raw_ptr_buffer_load_b8_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -74,7 +74,7 @@ u8 test_amdgcn_raw_ptr_buffer_load_b8_non_const_offset(__amdgpu_buffer_rsrc_t rs
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b16_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret i16 [[TMP0]]
//
u16 test_amdgcn_raw_ptr_buffer_load_b16_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -83,7 +83,7 @@ u16 test_amdgcn_raw_ptr_buffer_load_b16_non_const_offset(__amdgpu_buffer_rsrc_t
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret i32 [[TMP0]]
//
u32 test_amdgcn_raw_ptr_buffer_load_b32_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -92,7 +92,7 @@ u32 test_amdgcn_raw_ptr_buffer_load_b32_non_const_offset(__amdgpu_buffer_rsrc_t
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b64_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
v2u32 test_amdgcn_raw_ptr_buffer_load_b64_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -101,7 +101,7 @@ v2u32 test_amdgcn_raw_ptr_buffer_load_b64_non_const_offset(__amdgpu_buffer_rsrc_
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b96_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <3 x i32> [[TMP0]]
//
v3u32 test_amdgcn_raw_ptr_buffer_load_b96_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -110,7 +110,7 @@ v3u32 test_amdgcn_raw_ptr_buffer_load_b96_non_const_offset(__amdgpu_buffer_rsrc_
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b128_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_raw_ptr_buffer_load_b128_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -119,7 +119,7 @@ v4u32 test_amdgcn_raw_ptr_buffer_load_b128_non_const_offset(__amdgpu_buffer_rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b8_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret i8 [[TMP0]]
//
u8 test_amdgcn_raw_ptr_buffer_load_b8_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -128,7 +128,7 @@ u8 test_amdgcn_raw_ptr_buffer_load_b8_non_const_soffset(__amdgpu_buffer_rsrc_t r
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b16_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret i16 [[TMP0]]
//
u16 test_amdgcn_raw_ptr_buffer_load_b16_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -137,7 +137,7 @@ u16 test_amdgcn_raw_ptr_buffer_load_b16_non_const_soffset(__amdgpu_buffer_rsrc_t
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b32_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret i32 [[TMP0]]
//
u32 test_amdgcn_raw_ptr_buffer_load_b32_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -146,7 +146,7 @@ u32 test_amdgcn_raw_ptr_buffer_load_b32_non_const_soffset(__amdgpu_buffer_rsrc_t
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b64_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
v2u32 test_amdgcn_raw_ptr_buffer_load_b64_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -155,7 +155,7 @@ v2u32 test_amdgcn_raw_ptr_buffer_load_b64_non_const_soffset(__amdgpu_buffer_rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b96_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <3 x i32> [[TMP0]]
//
v3u32 test_amdgcn_raw_ptr_buffer_load_b96_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -164,7 +164,7 @@ v3u32 test_amdgcn_raw_ptr_buffer_load_b96_non_const_soffset(__amdgpu_buffer_rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_load_b128_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_raw_ptr_buffer_load_b128_non_const_soffset(__amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store-format.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store-format.cl
index 01e235c1d7024..0c25641c9d228 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store-format.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store-format.cl
@@ -9,7 +9,7 @@ typedef half v4f16 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_raw_buffer_store_format_v4f32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret void
//
void test_raw_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc) {
@@ -18,7 +18,7 @@ void test_raw_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc
// CHECK-LABEL: @test_raw_buffer_store_format_v4f16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_raw_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_rsrc_t rsrc) {
@@ -27,7 +27,7 @@ void test_raw_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_rsrc_t rsrc
// CHECK-LABEL: @test_raw_buffer_store_format_v4f32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_raw_buffer_store_format_v4f32_non_const_offset(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset) {
@@ -36,7 +36,7 @@ void test_raw_buffer_store_format_v4f32_non_const_offset(v4f32 vdata, __amdgpu_b
// CHECK-LABEL: @test_raw_buffer_store_format_v4f32_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_raw_buffer_store_format_v4f32_non_const_soffset(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc, int soffset) {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-load-format.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-load-format.cl
index fac5271f9ae26..291b604a4e752 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-load-format.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-load-format.cl
@@ -9,7 +9,7 @@ typedef half v4f16 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_struct_buffer_load_format_v4f32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
v4f32 test_struct_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc, int vindex) {
@@ -18,7 +18,7 @@ v4f32 test_struct_buffer_load_format_v4f32(__amdgpu_buffer_rsrc_t rsrc, int vind
// CHECK-LABEL: @test_struct_buffer_load_format_v4f16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x half> [[TMP0]]
//
v4f16 test_struct_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc, int vindex) {
@@ -27,7 +27,7 @@ v4f16 test_struct_buffer_load_format_v4f16(__amdgpu_buffer_rsrc_t rsrc, int vind
// CHECK-LABEL: @test_struct_buffer_load_format_v4f32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret <4 x float> [[TMP0]]
//
v4f32 test_struct_buffer_load_format_v4f32_non_const_offset(__amdgpu_buffer_rsrc_t rsrc, int vindex, int offset) {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-store-format.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-store-format.cl
index 7e04974c5e44c..abe093aa728e1 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-store-format.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-struct-buffer-store-format.cl
@@ -9,7 +9,7 @@ typedef half v4f16 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_struct_buffer_store_format_v4f32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret void
//
void test_struct_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc, int vindex) {
@@ -18,7 +18,7 @@ void test_struct_buffer_store_format_v4f32(v4f32 vdata, __amdgpu_buffer_rsrc_t r
// CHECK-LABEL: @test_struct_buffer_store_format_v4f16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f16(<4 x half> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_struct_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_rsrc_t rsrc, int vindex) {
@@ -27,7 +27,7 @@ void test_struct_buffer_store_format_v4f16(v4f16 vdata, __amdgpu_buffer_rsrc_t r
// CHECK-LABEL: @test_struct_buffer_store_format_v4f32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[VINDEX:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_struct_buffer_store_format_v4f32_non_const_offset(v4f32 vdata, __amdgpu_buffer_rsrc_t rsrc, int vindex, int offset) {
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 50caa5ba7e265..ac361ddfcaed2 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1545,6 +1545,36 @@ loads.
requirements of strict OOB mode. However, in strict OOB mode, those intrinsics
will not be combined in a way that would violate the guarantees of that mode.
+.. _amdgpu-buffer-intrinsic-atomicity:
+
+Buffer memory intrinsic atomicity
+---------------------------------
+
+The buffer memory intrinsics (``llvm.amdgcn.{raw,struct}.ptr.buffer.{load,store}``,
+``llvm.amdgcn.{raw,struct}.ptr.buffer.{load,store}.format``,
+``llvm.amdgcn.{raw,struct}.ptr.atomic.buffer.load`` and
+``llvm.amdgcn.{raw,struct}.ptr.buffer.atomic.*``) each take a trailing metadata
+argument that records the atomicity of the access. It is either the empty node,
+``!{}``, meaning that the access is not atomic, or a two-element node
+``!{!"<ordering>", !"<syncscope>"}``, where ``<ordering>`` is the LLVM IR
+spelling of the memory ordering (``unordered``, ``monotonic``, ``acquire``,
+``release``, ``acq_rel`` or ``seq_cst``) and ``<syncscope>`` is a synchronization
+scope name from table :ref:`amdgpu-amdhsa-llvm-sync-scopes-table` (the empty
+string denotes system scope).
+
+This argument is needed because a buffer instruction is the same instruction
+whether or not the access it implements is atomic, so the atomicity cannot be
+recovered from the selected instruction. The information is transferred to the
+``MachineMemOperand`` during instruction selection, where ``SIMemoryLegalizer``
+uses it to set the cache-bypass bits and to insert the cache invalidations and
+writebacks the memory model requires.
+
+``AMDGPULowerBufferFatPointers`` populates the argument from the ``atomicrmw``,
+``cmpxchg``, ``load`` or ``store`` instruction it lowers. Frontends that call
+these intrinsics directly and cannot express atomicity should pass ``!{}``.
+For compatibility, bitcode from before this argument existed is upgraded by
+appending ``!{}``.
+
Target Types
------------
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index b2862f1db0ea5..a95146eba525c 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1567,13 +1567,14 @@ class AMDGPURawPtrBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntri
[AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary/cachepolicy(imm):
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
// bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
// bit 3 = swz, bit 4 = scc (gfx90a)
// gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, IntrReadMem, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>,
ImmArg<ArgIndex<3>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
@@ -1589,7 +1590,7 @@ class AMDGPURawPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
// bit 1 = slc,
// bit 2 = dlc on gfx10+),
// swizzled buffer (bit 3 = swz))
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
def int_amdgcn_raw_ptr_atomic_buffer_load : AMDGPURawPtrAtomicBufferLoad;
@@ -1635,13 +1636,14 @@ class AMDGPUStructPtrBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIn
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary/cachepolicy(imm):
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
// bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
// bit 3 = swz, bit 4 = scc (gfx90a)
// gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, IntrReadMem, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>,
ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
@@ -1661,7 +1663,7 @@ class AMDGPUStructPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsi
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, NoCapture<ArgIndex<0>>,
ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
@@ -1688,23 +1690,6 @@ def int_amdgcn_raw_buffer_store : AMDGPURawBufferStore;
class AMDGPURawPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
[],
[data_ty, // vdata(VGPR)
- AMDGPUBufferRsrcTy, // rsrc(SGPR)
- llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
- llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary/cachepolicy(imm):
- // bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
- // bit 3 = swz, bit 4 = scc (gfx90a)
- // gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
- // gfx12+: bits [0-2] = th, bits [3-4] = scope,
- // bit 6 = swz
- // all: volatile op (bit 31, stripped at lowering)
- [IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
- ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
- AMDGPURsrcIntrinsic<1>;
-def int_amdgcn_raw_ptr_buffer_store_format : AMDGPURawPtrBufferStore<llvm_anyfloat_ty>;
-def int_amdgcn_raw_ptr_buffer_store : DefaultAttrsIntrinsic <
- [],
- [llvm_any_ty, // vdata(VGPR)
AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
@@ -1715,10 +1700,12 @@ def int_amdgcn_raw_ptr_buffer_store : DefaultAttrsIntrinsic <
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1>;
+def int_amdgcn_raw_ptr_buffer_store_format : AMDGPURawPtrBufferStore<llvm_anyfloat_ty>;
+def int_amdgcn_raw_ptr_buffer_store : AMDGPURawPtrBufferStore;
class AMDGPUStructBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
[],
@@ -1742,24 +1729,6 @@ def int_amdgcn_struct_buffer_store : AMDGPUStructBufferStore;
class AMDGPUStructPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
[],
[data_ty, // vdata(VGPR)
- AMDGPUBufferRsrcTy, // rsrc(SGPR)
- llvm_i32_ty, // vindex(VGPR)
- llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
- llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary/cachepolicy(imm):
- // bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
- // bit 3 = swz, bit 4 = scc (gfx90a)
- // gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
- // gfx12+: bits [0-2] = th, bits [3-4] = scope,
- // bit 6 = swz
- // all: volatile op (bit 31, stripped at lowering)
- [IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
- ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
- AMDGPURsrcIntrinsic<1>;
-def int_amdgcn_struct_ptr_buffer_store_format : AMDGPUStructPtrBufferStore;
-def int_amdgcn_struct_ptr_buffer_store : DefaultAttrsIntrinsic <
- [],
- [llvm_any_ty, // vdata(VGPR)
AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
@@ -1771,10 +1740,12 @@ def int_amdgcn_struct_ptr_buffer_store : DefaultAttrsIntrinsic <
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1>;
+def int_amdgcn_struct_ptr_buffer_store_format : AMDGPUStructPtrBufferStore;
+def int_amdgcn_struct_ptr_buffer_store : AMDGPUStructPtrBufferStore;
class AMDGPURawBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
[data_ty],
@@ -1819,7 +1790,7 @@ class AMDGPURawPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1, 0>;
@@ -1848,7 +1819,7 @@ def int_amdgcn_raw_ptr_buffer_atomic_cmpswap : Intrinsic<
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, NoCapture<ArgIndex<2>>,
ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<2, 0>;
@@ -1903,7 +1874,7 @@ class AMDGPUStructPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1, 0>;
@@ -1930,7 +1901,7 @@ def int_amdgcn_struct_ptr_buffer_atomic_cmpswap : Intrinsic<
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
- llvm_metadata_ty], // syncscope name (empty = no scope info)
+ llvm_metadata_ty], // atomicity: !{ordering, syncscope} or !{} if not atomic
[IntrArgMemOnly, NoCapture<ArgIndex<2>>,
ImmArg<ArgIndex<6>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<2, 0>;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 407d34bd03fc9..263b3abf6dd63 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1376,6 +1376,62 @@ static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
return true;
}
+// The intrinsics carrying a trailing atomicity metadata argument. Excludes the
+// tbuffer and buffer-to-LDS intrinsics, which do not.
+static bool isAMDGCNBufferMemIntrinsic(Intrinsic::ID IID) {
+ switch (IID) {
+ case Intrinsic::amdgcn_raw_ptr_buffer_load:
+ case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
+ case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
+ case Intrinsic::amdgcn_struct_ptr_buffer_load:
+ case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
+ case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
+ case Intrinsic::amdgcn_raw_ptr_buffer_store:
+ case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
+ case Intrinsic::amdgcn_struct_ptr_buffer_store:
+ case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
+ return true;
+ default:
+ return false;
+ }
+}
+
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
bool CanUpgradeDebugIntrinsicsToRecords) {
assert(F && "Illegal to upgrade a non-existent Function.");
@@ -1437,88 +1493,16 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return true;
}
break;
- // Legacy buffer atomic/store/atomic-load intrinsics missed the
- // trailing syncscope metadata operand.
- case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
- if (F->arg_size() == 4) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
- if (F->arg_size() == 5) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_raw_ptr_buffer_store:
- if (F->arg_size() == 5) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_struct_ptr_buffer_store:
- if (F->arg_size() == 6) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
- if (F->arg_size() == 5) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
- if (F->arg_size() == 6) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
- if (F->arg_size() == 6) {
- NewFn = nullptr;
- return true;
- }
- break;
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
- if (F->arg_size() == 7) {
- NewFn = nullptr;
- return true;
- }
- break;
+ }
+
+ // Detect the old form by the missing operand rather than by argument
+ // count, since the count differs per intrinsic.
+ if (isAMDGCNBufferMemIntrinsic(F->getIntrinsicID()) &&
+ !F->getFunctionType()
+ ->getParamType(F->arg_size() - 1)
+ ->isMetadataTy()) {
+ NewFn = nullptr;
+ return true;
}
if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
@@ -5019,10 +5003,9 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
}
- // Legacy buffer atomic/store/atomic-load intrinsics missed the trailing
- // syncscope metadata operand. Append an empty MDNode, meaning "no scope
- // info", matching the behavior of these calls before the operand existed.
- auto UpgradeBufferScopeIntrinsicCall =
+ // Before the trailing atomicity operand existed these calls were treated as
+ // not atomic, which the empty MDNode denotes.
+ auto AddNotAtomicMDToBufferCall =
[](Function *F, CallBase *CI, IRBuilder<> &Builder,
ArrayRef<Type *> OverloadTys) -> Value * {
SmallVector<Value *, 8> Args(CI->args());
@@ -5045,53 +5028,12 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
return NewCall;
};
- switch (F->getIntrinsicID()) {
- case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
- case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
- case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
- case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
- return UpgradeBufferScopeIntrinsicCall(F, CI, Builder,
- {F->getReturnType()});
- case Intrinsic::amdgcn_raw_ptr_buffer_store:
- case Intrinsic::amdgcn_struct_ptr_buffer_store:
- return UpgradeBufferScopeIntrinsicCall(F, CI, Builder,
- {CI->getArgOperand(0)->getType()});
- default:
- break;
+ if (isAMDGCNBufferMemIntrinsic(F->getIntrinsicID())) {
+ // Stores are overloaded on their data arg, everything else on the return.
+ Type *OverloadTy = F->getReturnType()->isVoidTy()
+ ? CI->getArgOperand(0)->getType()
+ : F->getReturnType();
+ return AddNotAtomicMDToBufferCall(F, CI, Builder, {OverloadTy});
}
switch (F->getIntrinsicID()) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index a12561e469fba..6fb53fdabfbce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6664,7 +6664,7 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
unsigned ImmOffset;
- // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // The trailing atomicity metadata operand on _ptr_ intrinsics is not one
// of the positional arguments counted below.
unsigned NumOperands = MI.getNumOperands();
if (MI.getOperand(NumOperands - 1).isMetadata())
@@ -6785,7 +6785,7 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
castBufferRsrcArgToV4I32(MI, B, 2 + OpOffset);
Register RSrc = MI.getOperand(2 + OpOffset).getReg();
- // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // The trailing atomicity metadata operand on _ptr_ intrinsics is not one
// of the positional arguments counted below.
unsigned NumOperands = MI.getNumOperands();
if (MI.getOperand(NumOperands - 1).isMetadata())
@@ -7067,7 +7067,7 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
Register RSrc = MI.getOperand(3 + OpOffset).getReg();
const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
- // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // The trailing atomicity metadata operand on _ptr_ intrinsics is not one
// of the positional arguments counted below.
unsigned NumOperands = MI.getNumOperands();
if (MI.getOperand(NumOperands - 1).isMetadata())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index def10425dd165..acb5c73acac3c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1566,9 +1566,8 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
- // Build the trailing syncscope metadata argument for the buffer
- // atomic/store/atomic-load intrinsics (empty string if not atomic).
- Value *getScopeMDArg(AtomicOrdering Order, SyncScope::ID SSID);
+ // Builds !{ordering, syncscope}, or !{} when the access is not atomic.
+ Value *getAtomicityMDArg(AtomicOrdering Order, SyncScope::ID SSID);
Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
Align Alignment, AtomicOrdering Order,
bool IsVolatile, SyncScope::ID SSID);
@@ -1875,15 +1874,15 @@ void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A));
}
-Value *SplitPtrStructs::getScopeMDArg(AtomicOrdering Order,
- SyncScope::ID SSID) {
+Value *SplitPtrStructs::getAtomicityMDArg(AtomicOrdering Order,
+ SyncScope::ID SSID) {
LLVMContext &Ctx = IRB.getContext();
- // An empty MDNode means not atomic. The system scope also has an empty
- // name, so scope presence cannot be encoded by string emptiness alone.
if (Order == AtomicOrdering::NotAtomic)
return MetadataAsValue::get(Ctx, MDNode::get(Ctx, {}));
StringRef Scope = Ctx.getSyncScopeName(SSID).value_or("");
- return MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDString::get(Ctx, Scope)));
+ Metadata *Ops[] = {MDString::get(Ctx, toIRString(Order)),
+ MDString::get(Ctx, Scope)};
+ return MetadataAsValue::get(Ctx, MDNode::get(Ctx, Ops));
}
void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
@@ -1935,19 +1934,12 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
Aux |= AMDGPU::CPol::VOLATILE;
Args.push_back(IRB.getInt32(Aux));
- // amdgcn_raw_ptr_buffer_load has no scope operand, unlike the atomic load,
- // store, and RMW intrinsics used below.
- bool NeedsScopeArg = true;
-
Intrinsic::ID IID = Intrinsic::not_intrinsic;
- if (isa<LoadInst>(I)) {
- if (Order == AtomicOrdering::NotAtomic) {
- IID = Intrinsic::amdgcn_raw_ptr_buffer_load;
- NeedsScopeArg = false;
- } else {
- IID = Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
- }
- } else if (isa<StoreInst>(I))
+ if (isa<LoadInst>(I))
+ IID = Order == AtomicOrdering::NotAtomic
+ ? Intrinsic::amdgcn_raw_ptr_buffer_load
+ : Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
+ else if (isa<StoreInst>(I))
IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) {
switch (RMW->getOperation()) {
@@ -2042,8 +2034,7 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
}
}
- if (NeedsScopeArg)
- Args.push_back(getScopeMDArg(Order, SSID));
+ Args.push_back(getAtomicityMDArg(Order, SSID));
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(IID, Ty, Args);
copyMetadata(Call, I);
@@ -2115,7 +2106,7 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(
Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, Ty,
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
- IRB.getInt32(0), IRB.getInt32(Aux), getScopeMDArg(Order, SSID)});
+ IRB.getInt32(0), IRB.getInt32(Aux), getAtomicityMDArg(Order, SSID)});
copyMetadata(Call, &AI);
setAlign(Call, AI.getAlign(), 2);
Call->takeName(&AI);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 17132c8b5b863..4755e7da92a55 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1422,23 +1422,34 @@ static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
return CI.getContext().getOrInsertSyncScopeID(Scope);
}
-// Buffer atomic/store/atomic-load intrinsics take the syncscope name as
-// their trailing metadata argument, an empty MDNode when the memory access
-// is not atomic (the system scope also has an empty name, so the MDNode
-// operand count, not string emptiness, distinguishes the two). Not every
-// intrinsic reaching this dispatch has that trailing argument, so callers
-// rely on this returning std::nullopt for those too.
-static std::optional<SyncScope::ID>
-parseBufferAtomicScopeArg(const CallBase &CI) {
- auto *ScopeArg =
- dyn_cast<MetadataAsValue>(CI.getArgOperand(CI.arg_size() - 1));
- if (!ScopeArg)
+// A buffer instruction is identical whether or not the access is atomic, so
+// the trailing !{ordering, syncscope} argument is the only record of it.
+// Returns std::nullopt for a non-atomic access, and for the intrinsics
+// reaching this dispatch that have no such trailing argument at all.
+static std::optional<std::pair<AtomicOrdering, SyncScope::ID>>
+parseBufferAtomicityMDArg(const CallBase &CI) {
+ auto *MDArg = dyn_cast<MetadataAsValue>(CI.getArgOperand(CI.arg_size() - 1));
+ if (!MDArg)
return std::nullopt;
- MDNode *ScopeMD = cast<MDNode>(ScopeArg->getMetadata());
- if (ScopeMD->getNumOperands() == 0)
+ auto *MD = cast<MDNode>(MDArg->getMetadata());
+ if (MD->getNumOperands() == 0)
return std::nullopt;
- StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
- return CI.getContext().getOrInsertSyncScopeID(Scope);
+
+ StringRef OrderStr = cast<MDString>(MD->getOperand(0))->getString();
+ AtomicOrdering Order =
+ StringSwitch<AtomicOrdering>(OrderStr)
+ .Case("unordered", AtomicOrdering::Unordered)
+ .Case("monotonic", AtomicOrdering::Monotonic)
+ .Case("acquire", AtomicOrdering::Acquire)
+ .Case("release", AtomicOrdering::Release)
+ .Case("acq_rel", AtomicOrdering::AcquireRelease)
+ .Case("seq_cst", AtomicOrdering::SequentiallyConsistent)
+ .Default(AtomicOrdering::NotAtomic);
+ if (Order == AtomicOrdering::NotAtomic)
+ return std::nullopt;
+
+ StringRef Scope = cast<MDString>(MD->getOperand(1))->getString();
+ return std::make_pair(Order, CI.getContext().getOrInsertSyncScopeID(Scope));
}
void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
@@ -1462,7 +1473,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
if (!IsSPrefetch) {
- // Some buffer intrinsics have a trailing syncscope metadata arg after
+ // Buffer memory intrinsics have a trailing atomicity metadata arg after
// the cachepolicy immarg.
unsigned AuxIdx = CI.arg_size() - 1;
if (isa<MetadataAsValue>(CI.getArgOperand(AuxIdx)))
@@ -1533,12 +1544,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = getValueType(MF.getDataLayout(), DataTy);
Info.flags = Flags | MachineMemOperand::MOStore;
- if (std::optional<SyncScope::ID> SSID = parseBufferAtomicScopeArg(CI)) {
- // Pretend to be atomic so SIMemoryLegalizer::expandStore sets cache
- // bypass bits, since atomic and plain buffer stores look identical.
- Info.order = AtomicOrdering::Monotonic;
- Info.ssid = *SSID;
- }
+ if (auto Atomicity = parseBufferAtomicityMDArg(CI))
+ std::tie(Info.order, Info.ssid) = *Atomicity;
} else {
// Atomic, NoReturn Sampler or prefetch
Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
@@ -1557,13 +1564,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// XXX - Should this be volatile without known ordering?
Info.flags |= MachineMemOperand::MOVolatile;
Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
- if (std::optional<SyncScope::ID> SSID =
- parseBufferAtomicScopeArg(CI)) {
- // Pretend to be atomic so expandAtomicCmpxchgOrRmw sets cache
- // bypass bits.
- Info.order = AtomicOrdering::Monotonic;
- Info.ssid = *SSID;
- }
+ if (auto Atomicity = parseBufferAtomicityMDArg(CI))
+ std::tie(Info.order, Info.ssid) = *Atomicity;
}
break;
case Intrinsic::amdgcn_raw_buffer_load_lds:
@@ -1604,11 +1606,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags = Flags | MachineMemOperand::MOLoad;
- if (std::optional<SyncScope::ID> SSID = parseBufferAtomicScopeArg(CI)) {
- // Pretend to be atomic so expandLoad sets cache bypass bits.
- Info.order = AtomicOrdering::Monotonic;
- Info.ssid = *SSID;
- }
+ if (auto Atomicity = parseBufferAtomicityMDArg(CI))
+ std::tie(Info.order, Info.ssid) = *Atomicity;
Infos.push_back(Info);
return;
}
diff --git a/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll b/llvm/test/Bitcode/amdgpu-buffer-atomicity-upgrade.ll
similarity index 67%
rename from llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
rename to llvm/test/Bitcode/amdgpu-buffer-atomicity-upgrade.ll
index 3f9550e026c8e..f1627204154bf 100644
--- a/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
+++ b/llvm/test/Bitcode/amdgpu-buffer-atomicity-upgrade.ll
@@ -1,28 +1,40 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+declare float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32 immarg)
+declare <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8), i32, i32, i32 immarg)
declare void @llvm.amdgcn.raw.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float>, ptr addrspace(8), i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8), i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32, i32, ptr addrspace(8), i32, i32, i32 immarg)
+declare float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32, i32 immarg)
declare void @llvm.amdgcn.struct.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32, i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
-define void @test(i32 %v, ptr addrspace(8) %rsrc, float %f) {
+define void @test(i32 %v, ptr addrspace(8) %rsrc, float %f, <4 x float> %f4) {
; CHECK-LABEL: define void @test(
+; CHECK: call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> %f4, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
; CHECK: call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+ %l = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %lf = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.raw.ptr.buffer.store.format.v4f32(<4 x float> %f4, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%a = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%b = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%c = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %sl = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%d = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%e = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index 7fc4b3fbd9a50..9041ab8dc82e4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -10,7 +10,7 @@ declare void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32>, <4 x i32>,
declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32, ptr addrspace (8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
-declare void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32>, ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32>, ptr addrspace(8), i32, i32, i32, i32 immarg, metadata)
define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
@@ -370,7 +370,7 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP5]]
; IR-NEXT: [[ARG_INT:%.*]] = ptrtoint ptr addrspace(8) [[ARG]] to i128
; IR-NEXT: [[ARG_VEC:%.*]] = bitcast i128 [[ARG_INT]] to <4 x i32>
-; IR-NEXT: call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> [[ARG_VEC]], ptr addrspace(8) [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0)
+; IR-NEXT: call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> [[ARG_VEC]], ptr addrspace(8) [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; IR-NEXT: ret void
;
; GCN-LABEL: atomic_ptr_add_and_format:
@@ -403,7 +403,7 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
%a = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
%arg.int = ptrtoint ptr addrspace(8) %arg to i128
%arg.vec = bitcast i128 %arg.int to <4 x i32>
- call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> %arg.vec, ptr addrspace(8) %arg, i32 %a, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> %arg.vec, ptr addrspace(8) %arg, i32 %a, i32 0, i32 0, i32 0, metadata !0)
ret void
}
@@ -470,7 +470,7 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], [[TMP5]]
; IR-NEXT: [[ARG_INT:%.*]] = ptrtoint ptr addrspace(8) [[ARG]] to i128
; IR-NEXT: [[ARG_VEC:%.*]] = bitcast i128 [[ARG_INT]] to <4 x i32>
-; IR-NEXT: call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> [[ARG_VEC]], ptr addrspace(8) [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0)
+; IR-NEXT: call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> [[ARG_VEC]], ptr addrspace(8) [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; IR-NEXT: ret void
;
; GCN-LABEL: atomic_ptr_sub_and_format:
@@ -503,7 +503,7 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
%a = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
%arg.int = ptrtoint ptr addrspace(8) %arg to i128
%arg.vec = bitcast i128 %arg.int to <4 x i32>
- call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> %arg.vec, ptr addrspace(8) %arg, i32 %a, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> %arg.vec, ptr addrspace(8) %arg, i32 %a, i32 0, i32 0, i32 0, metadata !0)
ret void
}
@@ -574,7 +574,7 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP16:%.*]] = xor i32 [[TMP14]], [[TMP15]]
; IR-NEXT: [[ARG_INT:%.*]] = ptrtoint ptr addrspace(8) [[ARG]] to i128
; IR-NEXT: [[ARG_VEC:%.*]] = bitcast i128 [[ARG_INT]] to <4 x i32>
-; IR-NEXT: call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> [[ARG_VEC]], ptr addrspace(8) [[ARG]], i32 [[TMP16]], i32 0, i32 0, i32 0)
+; IR-NEXT: call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> [[ARG_VEC]], ptr addrspace(8) [[ARG]], i32 [[TMP16]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; IR-NEXT: ret void
;
; GCN-LABEL: atomic_ptr_xor_and_format:
@@ -609,6 +609,8 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
%a = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
%arg.int = ptrtoint ptr addrspace(8) %arg to i128
%arg.vec = bitcast i128 %arg.int to <4 x i32>
- call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> %arg.vec, ptr addrspace(8) %arg, i32 %a, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32> %arg.vec, ptr addrspace(8) %arg, i32 %a, i32 0, i32 0, i32 0, metadata !0)
ret void
}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
index c7f2ec5d96663..60cb35b96a134 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
@@ -1500,7 +1500,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1526,9 +1528,12 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1546,6 +1551,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1557,6 +1564,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1580,6 +1589,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1610,6 +1620,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1639,6 +1650,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1654,6 +1666,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr addrspace(7) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
@@ -1669,9 +1682,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1693,9 +1708,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -1714,6 +1732,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1725,6 +1745,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1746,6 +1768,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -1775,6 +1798,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -1804,6 +1828,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -1820,6 +1845,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fmax ptr addrspace(7) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
@@ -1854,7 +1880,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1873,9 +1901,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1906,6 +1937,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1925,6 +1958,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1934,6 +1969,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1962,6 +1998,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1998,6 +2035,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2015,6 +2053,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr addrspace(7) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret double %result
@@ -2048,7 +2087,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
@@ -2067,9 +2108,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2099,6 +2143,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2118,6 +2164,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2127,6 +2175,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[16:19], 0 offen
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2153,6 +2202,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2187,6 +2237,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2204,6 +2255,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fmax ptr addrspace(7) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
index d06c52c2d5211..65a6753a79f42 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
@@ -1500,7 +1500,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1526,9 +1528,12 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1546,6 +1551,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1557,6 +1564,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1580,6 +1589,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1610,6 +1620,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1639,6 +1650,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1654,6 +1666,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr addrspace(7) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret float %result
@@ -1669,9 +1682,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1693,9 +1708,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -1714,6 +1732,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1725,6 +1745,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1746,6 +1768,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -1775,6 +1798,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -1804,6 +1828,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -1820,6 +1845,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fmin ptr addrspace(7) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
@@ -1854,7 +1880,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1873,9 +1901,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1906,6 +1937,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1925,6 +1958,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1934,6 +1969,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1962,6 +1998,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1998,6 +2035,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2015,6 +2053,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr addrspace(7) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret double %result
@@ -2048,7 +2087,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
@@ -2067,9 +2108,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2099,6 +2143,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2118,6 +2164,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2127,6 +2175,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2153,6 +2202,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2187,6 +2237,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2204,6 +2255,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fmin ptr addrspace(7) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 0cc2d8b06d254..e29c243763daa 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -25,7 +25,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -36,9 +38,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -50,6 +55,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -72,6 +79,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -87,6 +96,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -107,6 +117,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -133,6 +144,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -159,6 +171,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -187,6 +200,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -210,9 +224,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -221,9 +237,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -235,6 +254,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -255,6 +276,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -271,6 +294,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -280,6 +304,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -298,6 +323,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -323,6 +349,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v3
@@ -350,6 +377,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v2, v3
@@ -385,16 +413,17 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -415,15 +444,17 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX942-NEXT: ; implicit-def: $vgpr4
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -442,15 +473,16 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -495,14 +527,15 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -529,15 +562,16 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_add_f32 v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, v5
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -580,13 +614,14 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -636,13 +671,14 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB2_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -691,13 +727,13 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB2_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v8, v6
@@ -748,13 +784,14 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB2_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v8, v6
@@ -781,7 +818,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -792,9 +831,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -806,6 +848,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -828,6 +872,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -853,6 +899,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -879,6 +926,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -905,6 +953,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -931,6 +980,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -959,6 +1009,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -982,9 +1033,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -993,9 +1046,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -1007,6 +1063,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -1027,6 +1085,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1051,6 +1111,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -1076,6 +1137,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -1101,6 +1163,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -1126,6 +1189,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v3
@@ -1153,6 +1217,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v2, v3
@@ -1177,7 +1242,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1188,9 +1255,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:
@@ -1212,6 +1282,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1241,6 +1313,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -1266,6 +1340,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1292,6 +1367,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1318,6 +1394,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1344,6 +1421,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1372,6 +1450,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1395,7 +1474,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1406,9 +1487,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:
@@ -1430,6 +1514,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1459,6 +1545,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -1484,6 +1572,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1510,6 +1599,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1536,6 +1626,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1562,6 +1653,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1590,6 +1682,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1613,7 +1706,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1624,9 +1719,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:
@@ -1648,6 +1746,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1677,6 +1777,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -1702,6 +1804,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1728,6 +1831,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1754,6 +1858,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1780,6 +1885,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1808,6 +1914,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1847,7 +1954,9 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
; GFX12-NEXT: v_mov_b32_e32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1866,9 +1975,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1892,6 +2004,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1924,6 +2038,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
@@ -1941,6 +2057,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1963,6 +2080,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1993,6 +2111,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2023,6 +2142,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2055,6 +2175,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2090,7 +2211,9 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
@@ -2109,9 +2232,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -2133,6 +2259,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2163,6 +2291,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v4, v6
; GFX10-NEXT: v_mov_b32_e32 v5, v7
@@ -2180,6 +2310,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -2200,6 +2331,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2228,6 +2360,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2256,6 +2389,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2286,6 +2420,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX6-NEXT: v_mov_b32_e32 v4, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2353,15 +2488,16 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB10_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2389,15 +2525,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX942-NEXT: ; implicit-def: $vgpr4
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: v_mov_b32_e32 v1, v7
; GFX942-NEXT: buffer_inv sc1
@@ -2446,13 +2584,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB10_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
@@ -2510,14 +2649,15 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB10_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX10-NEXT: v_mov_b32_e32 v14, v1
; GFX10-NEXT: v_mov_b32_e32 v13, v0
@@ -2546,15 +2686,16 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_add_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: buffer_wbinvl1
@@ -2604,13 +2745,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX908-NEXT: v_mov_b32_e32 v14, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -2666,13 +2808,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX8-NEXT: v_mov_b32_e32 v14, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -2727,13 +2870,13 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB10_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX7-NEXT: v_mov_b32_e32 v14, v1
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -2791,13 +2934,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB10_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX6-NEXT: v_mov_b32_e32 v14, v1
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -2836,7 +2980,9 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
; GFX12-NEXT: v_mov_b32_e32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -2855,9 +3001,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:
@@ -2881,6 +3030,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2913,6 +3064,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
@@ -2941,6 +3094,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]
@@ -2970,6 +3124,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3000,6 +3155,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3030,6 +3186,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3062,6 +3219,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3099,7 +3257,9 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
; GFX12-NEXT: v_mov_b32_e32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -3118,9 +3278,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -3144,6 +3307,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -3176,6 +3341,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
@@ -3193,6 +3360,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -3215,6 +3383,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3245,6 +3414,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3275,6 +3445,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3307,6 +3478,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -3361,7 +3533,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -3409,7 +3583,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -3446,9 +3622,12 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -3491,6 +3670,8 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3535,6 +3716,8 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3573,6 +3756,8 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3607,6 +3792,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -3642,6 +3828,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -3678,6 +3865,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -3717,6 +3905,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
@@ -3757,6 +3946,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
@@ -3807,7 +3997,9 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -3854,7 +4046,9 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -3890,9 +4084,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -3934,6 +4131,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3977,6 +4176,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4014,6 +4215,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4047,6 +4250,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -4081,6 +4285,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -4116,6 +4321,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -4154,6 +4360,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
@@ -4193,6 +4400,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v3
@@ -4268,15 +4476,16 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4347,15 +4556,16 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4416,13 +4626,15 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v7, v8
@@ -4487,13 +4699,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -4560,13 +4773,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -4628,14 +4842,15 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: buffer_gl1_inv
@@ -4696,13 +4911,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB15_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v7, v8
@@ -4762,13 +4978,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v7, v8
@@ -4829,13 +5046,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v7, v8
@@ -4897,13 +5115,13 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v6, v4
@@ -4965,13 +5183,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v6, v4
@@ -5036,7 +5255,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
@@ -5080,9 +5301,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v2
@@ -5133,6 +5357,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v2
; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -5177,6 +5403,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -5218,6 +5446,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
@@ -5260,6 +5489,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
@@ -5304,6 +5534,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
@@ -5345,6 +5576,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
@@ -5387,6 +5619,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
@@ -5446,7 +5679,9 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
@@ -5489,9 +5724,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v4
@@ -5541,6 +5779,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v3
; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -5584,6 +5824,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -5624,6 +5866,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
@@ -5665,6 +5908,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v3
@@ -5708,6 +5952,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -5748,6 +5993,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
@@ -5789,6 +6035,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v3
@@ -5874,15 +6121,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5950,13 +6198,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB18_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v7, v4
@@ -6031,13 +6281,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: buffer_gl1_inv
@@ -6106,14 +6357,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
@@ -6181,13 +6433,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB18_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
@@ -6254,13 +6507,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v6, v4
@@ -6328,13 +6582,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v6, v4
@@ -6398,13 +6653,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v5, v6
@@ -6469,13 +6724,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v5, v6
@@ -6506,7 +6762,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -6517,9 +6775,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6545,6 +6806,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -6580,6 +6843,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -6615,6 +6880,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -6630,6 +6897,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6655,6 +6923,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6688,6 +6957,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6727,6 +6997,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6770,6 +7041,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -6797,9 +7069,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6808,9 +7082,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6834,6 +7111,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6868,6 +7147,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6902,6 +7183,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6918,6 +7201,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6927,6 +7211,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX908-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6952,6 +7237,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -6992,6 +7278,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -7032,6 +7319,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -7067,16 +7355,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -7097,15 +7386,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX942-NEXT: ; implicit-def: $vgpr4
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7166,13 +7457,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -7242,13 +7534,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -7318,14 +7611,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -7352,15 +7646,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, v5
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7421,13 +7716,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -7497,13 +7793,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -7580,13 +7877,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -7668,13 +7965,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_6
; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -7703,7 +8001,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -7714,9 +8014,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
@@ -7742,6 +8045,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7777,6 +8082,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7812,6 +8119,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -7842,6 +8151,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -7873,6 +8183,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -7906,6 +8217,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -7945,6 +8257,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -7988,6 +8301,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -8015,9 +8329,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
@@ -8026,9 +8342,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
@@ -8052,6 +8371,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8086,6 +8407,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8120,6 +8443,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8149,6 +8474,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -8179,6 +8505,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -8211,6 +8538,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -8251,6 +8579,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8291,6 +8620,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -8315,7 +8645,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -8326,9 +8658,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
@@ -8354,6 +8689,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -8389,6 +8726,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -8424,6 +8763,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -8454,6 +8795,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -8485,6 +8827,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -8518,6 +8861,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -8557,6 +8901,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8600,6 +8945,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -8627,9 +8973,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
@@ -8638,9 +8986,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
@@ -8664,6 +9015,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8698,6 +9051,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8732,6 +9087,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8761,6 +9118,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -8791,6 +9149,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -8823,6 +9182,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -8863,6 +9223,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8903,6 +9264,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -8931,7 +9293,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -8975,9 +9339,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -9033,6 +9400,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -9090,6 +9459,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -9142,6 +9513,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
@@ -9190,6 +9563,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -9239,6 +9613,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -9289,6 +9664,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -9333,6 +9709,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -9384,6 +9761,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -9413,9 +9791,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
@@ -9455,9 +9835,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -9513,6 +9896,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -9570,6 +9955,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -9621,6 +10008,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -9668,6 +10057,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -9716,6 +10106,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -9765,6 +10156,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -9810,6 +10202,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -9857,6 +10250,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -9893,16 +10287,17 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -9984,13 +10379,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB28_6
; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -10080,13 +10477,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -10175,13 +10573,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -10267,14 +10666,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -10361,13 +10761,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB28_6
; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -10453,13 +10854,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB28_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -10546,13 +10948,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB28_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -10633,13 +11036,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB28_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -10726,13 +11129,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB28_6
; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -10763,7 +11167,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -10807,9 +11213,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -10865,6 +11274,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -10922,6 +11333,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -10974,6 +11387,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
@@ -11022,6 +11437,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -11071,6 +11487,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -11121,6 +11538,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -11165,6 +11583,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -11216,6 +11635,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -11245,9 +11665,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
@@ -11287,9 +11709,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -11345,6 +11770,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -11402,6 +11829,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -11453,6 +11882,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -11500,6 +11931,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -11548,6 +11980,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -11597,6 +12030,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -11642,6 +12076,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -11689,6 +12124,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -11714,7 +12150,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -11758,9 +12196,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -11816,6 +12257,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -11873,6 +12316,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -11925,6 +12370,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
@@ -11973,6 +12420,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -12022,6 +12470,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -12072,6 +12521,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -12116,6 +12566,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -12167,6 +12618,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -12196,9 +12648,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
@@ -12238,9 +12692,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -12296,6 +12753,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -12353,6 +12812,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -12404,6 +12865,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -12451,6 +12914,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -12499,6 +12963,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -12548,6 +13013,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -12593,6 +13059,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -12640,6 +13107,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -12665,9 +13133,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -12707,9 +13177,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -12765,6 +13238,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -12822,6 +13297,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -12873,6 +13350,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -12920,6 +13399,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -12968,6 +13448,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -13017,6 +13498,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -13062,6 +13544,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -13109,6 +13592,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -13139,7 +13623,11 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -13150,8 +13638,11 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc0 sc1
; GFX942-NEXT: buffer_inv sc0 sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -13164,6 +13655,8 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -13186,6 +13679,8 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -13210,10 +13705,14 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -13240,6 +13739,7 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -13266,6 +13766,7 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -13292,6 +13793,7 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -13320,6 +13822,7 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 54efdf290801c..c45420d2925f2 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -25,7 +25,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -48,9 +50,12 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -68,6 +73,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -79,6 +86,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -100,6 +109,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -128,6 +138,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -156,6 +167,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -171,6 +183,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -180,6 +193,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
@@ -197,9 +211,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -218,9 +234,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -239,6 +258,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -250,6 +271,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -269,6 +292,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -296,6 +320,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -323,6 +348,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -339,6 +365,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -348,6 +375,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
@@ -376,16 +404,17 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -432,13 +461,15 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -464,15 +495,16 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -492,8 +524,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
@@ -501,7 +535,6 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s8
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
@@ -546,13 +579,14 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB2_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -604,13 +638,14 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -662,13 +697,14 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB2_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v7, v5
@@ -693,15 +729,15 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB2_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, v5
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -719,15 +755,16 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB2_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v5
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -747,7 +784,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -770,9 +809,12 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -804,6 +846,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -835,6 +879,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -862,6 +908,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -890,6 +937,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -918,6 +966,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -946,6 +995,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -976,6 +1026,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -999,7 +1050,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1022,9 +1075,12 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1042,6 +1098,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1053,6 +1111,8 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1074,6 +1134,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1102,6 +1163,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1130,6 +1192,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1145,6 +1208,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1154,6 +1218,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
@@ -1188,7 +1253,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1207,9 +1274,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1235,6 +1305,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1254,6 +1326,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1263,6 +1337,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1287,6 +1362,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1319,6 +1395,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1336,6 +1413,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1345,6 +1423,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
@@ -1375,7 +1454,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
@@ -1394,9 +1475,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1422,6 +1506,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1441,6 +1527,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1450,6 +1538,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1472,6 +1561,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1502,6 +1592,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1519,6 +1610,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1528,6 +1620,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
@@ -1589,15 +1682,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB7_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1625,15 +1719,17 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX942-NEXT: ; implicit-def: $vgpr4
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: v_mov_b32_e32 v1, v7
; GFX942-NEXT: buffer_inv sc1
@@ -1684,13 +1780,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
@@ -1717,8 +1814,10 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
@@ -1726,7 +1825,6 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s8
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: v_mov_b32_e32 v1, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -1748,15 +1846,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: buffer_wbinvl1
@@ -1808,13 +1907,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX908-NEXT: v_mov_b32_e32 v14, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -1872,13 +1972,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX8-NEXT: v_mov_b32_e32 v14, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -1903,15 +2004,15 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB7_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, v5
; GFX7-NEXT: v_mov_b32_e32 v1, v6
; GFX7-NEXT: buffer_wbinvl1
@@ -1930,15 +2031,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v5
; GFX6-NEXT: v_mov_b32_e32 v1, v6
; GFX6-NEXT: buffer_wbinvl1
@@ -1972,7 +2074,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1991,9 +2095,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
@@ -2019,6 +2126,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2053,6 +2162,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
@@ -2083,6 +2194,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]
@@ -2114,6 +2226,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2146,6 +2259,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2178,6 +2292,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2212,6 +2327,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2250,7 +2366,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -2269,9 +2387,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2297,6 +2418,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2316,6 +2439,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2325,6 +2450,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2349,6 +2475,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2381,6 +2508,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2398,6 +2526,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2407,6 +2536,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
@@ -2455,7 +2585,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -2505,7 +2637,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -2544,9 +2678,12 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -2592,6 +2729,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -2638,6 +2777,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -2678,6 +2819,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -2714,6 +2857,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -2751,6 +2895,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -2789,6 +2934,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -2828,6 +2974,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
@@ -2868,6 +3015,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
@@ -2920,7 +3068,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -2969,7 +3119,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -3007,9 +3159,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -3054,6 +3209,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3099,6 +3256,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3138,6 +3297,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3173,6 +3334,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -3209,6 +3371,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -3246,6 +3409,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -3284,6 +3448,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
@@ -3323,6 +3488,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v3
@@ -3401,15 +3567,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3483,15 +3650,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3554,13 +3722,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v7, v8
@@ -3627,13 +3797,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -3702,13 +3873,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -3772,14 +3944,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: buffer_gl1_inv
@@ -3842,13 +4015,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v7, v8
@@ -3910,13 +4084,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v7, v8
@@ -3979,13 +4154,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v7, v8
@@ -4047,13 +4223,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB12_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v6, v4
@@ -4115,13 +4291,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v6, v4
@@ -4186,7 +4363,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
@@ -4230,9 +4409,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v2
@@ -4283,6 +4465,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v2
; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4327,6 +4511,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4368,6 +4554,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
@@ -4410,6 +4597,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
@@ -4454,6 +4642,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
@@ -4496,6 +4685,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
@@ -4539,6 +4729,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
@@ -4598,7 +4789,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
@@ -4641,9 +4834,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v4
@@ -4693,6 +4889,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v3
; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4736,6 +4934,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4776,6 +4976,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
@@ -4817,6 +5018,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v3
@@ -4860,6 +5062,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -4901,6 +5104,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
@@ -4943,6 +5147,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v3
@@ -5028,15 +5233,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5104,13 +5310,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v7, v4
@@ -5185,13 +5393,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: buffer_gl1_inv
@@ -5260,14 +5469,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
@@ -5335,13 +5545,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB15_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
@@ -5408,13 +5619,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v6, v4
@@ -5482,13 +5694,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v6, v4
@@ -5553,13 +5766,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v5, v6
@@ -5625,13 +5838,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v5, v6
@@ -5678,7 +5892,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v3, v1, v0
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -5718,7 +5934,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v3, v1, v0
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -5753,9 +5971,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -5789,6 +6010,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5826,6 +6049,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5863,6 +6088,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -5895,6 +6122,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -5928,6 +6156,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -5965,6 +6194,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6004,6 +6234,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6047,6 +6278,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -6089,7 +6321,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -6129,7 +6363,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -6163,9 +6399,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -6199,6 +6438,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6236,6 +6477,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6272,6 +6515,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6303,6 +6548,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -6335,6 +6581,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -6371,6 +6618,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -6411,6 +6659,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6451,6 +6700,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -6536,15 +6786,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6625,15 +6876,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6704,13 +6956,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB18_6
; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -6781,13 +7035,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -6860,13 +7115,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -6938,14 +7194,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -7016,13 +7273,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB18_6
; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -7092,13 +7350,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -7172,13 +7431,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -7255,13 +7515,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -7343,13 +7603,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_6
; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -7419,7 +7680,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -7478,7 +7741,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -7529,9 +7794,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7587,6 +7855,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7644,6 +7914,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7696,6 +7968,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
@@ -7744,6 +8018,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7793,6 +8068,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7843,6 +8119,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7887,6 +8164,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -7938,6 +8216,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8003,7 +8282,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -8062,7 +8343,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -8112,9 +8395,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -8170,6 +8456,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8227,6 +8515,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -8278,6 +8568,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -8325,6 +8617,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -8373,6 +8666,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -8422,6 +8716,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -8467,6 +8762,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8514,6 +8810,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8621,15 +8918,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8728,15 +9026,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8824,13 +9123,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_6
; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -8920,13 +9221,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -9015,13 +9317,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -9107,14 +9410,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -9201,13 +9505,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_6
; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -9293,13 +9598,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -9386,13 +9692,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -9473,13 +9780,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -9567,13 +9874,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_6
; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -9609,7 +9917,11 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -9632,8 +9944,11 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc0 sc1
; GFX942-NEXT: buffer_inv sc0 sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -9652,6 +9967,8 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -9663,6 +9980,8 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -9683,10 +10002,14 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -9715,6 +10038,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -9743,6 +10067,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -9758,6 +10083,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -9767,6 +10093,7 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmax v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index f4512c9ac18ce..9d8034136d40c 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -25,7 +25,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -48,9 +50,12 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -68,6 +73,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -79,6 +86,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -100,6 +109,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -128,6 +138,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -156,6 +167,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -171,6 +183,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -180,6 +193,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
@@ -197,9 +211,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -218,9 +234,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -239,6 +258,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -250,6 +271,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -269,6 +292,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -296,6 +320,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -323,6 +348,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -339,6 +365,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -348,6 +375,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
@@ -376,16 +404,17 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -432,13 +461,15 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -464,15 +495,16 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -492,8 +524,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
@@ -501,7 +535,6 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s8
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
@@ -546,13 +579,14 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB2_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -604,13 +638,14 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -662,13 +697,14 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB2_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v7, v5
@@ -693,15 +729,15 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB2_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, v5
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -719,15 +755,16 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB2_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v5
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -747,7 +784,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -770,9 +809,12 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -804,6 +846,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -835,6 +879,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -862,6 +908,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -890,6 +937,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -918,6 +966,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -946,6 +995,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -976,6 +1026,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -999,7 +1050,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1022,9 +1075,12 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1042,6 +1098,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1053,6 +1111,8 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1074,6 +1134,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1102,6 +1163,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1130,6 +1192,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1145,6 +1208,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1154,6 +1218,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
@@ -1188,7 +1253,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1207,9 +1274,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1235,6 +1305,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1254,6 +1326,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1263,6 +1337,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1287,6 +1362,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1319,6 +1395,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1336,6 +1413,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1345,6 +1423,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
@@ -1375,7 +1454,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
@@ -1394,9 +1475,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1422,6 +1506,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -1441,6 +1527,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1450,6 +1538,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1472,6 +1561,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1502,6 +1592,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1519,6 +1610,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1528,6 +1620,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
@@ -1589,15 +1682,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB7_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1625,15 +1719,17 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX942-NEXT: ; implicit-def: $vgpr4
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: v_mov_b32_e32 v1, v7
; GFX942-NEXT: buffer_inv sc1
@@ -1684,13 +1780,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
@@ -1717,8 +1814,10 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
@@ -1726,7 +1825,6 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s8
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: v_mov_b32_e32 v1, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -1748,15 +1846,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX90A-NEXT: ; implicit-def: $vgpr4
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: buffer_wbinvl1
@@ -1808,13 +1907,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX908-NEXT: v_mov_b32_e32 v14, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -1872,13 +1972,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
; GFX8-NEXT: v_mov_b32_e32 v14, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -1903,15 +2004,15 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX7-NEXT: ; implicit-def: $vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB7_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, v5
; GFX7-NEXT: v_mov_b32_e32 v1, v6
; GFX7-NEXT: buffer_wbinvl1
@@ -1930,15 +2031,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v5
; GFX6-NEXT: v_mov_b32_e32 v1, v6
; GFX6-NEXT: buffer_wbinvl1
@@ -1972,7 +2074,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -1991,9 +2095,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
@@ -2019,6 +2126,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2053,6 +2162,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
@@ -2083,6 +2194,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]
@@ -2114,6 +2226,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2146,6 +2259,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2178,6 +2292,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2212,6 +2327,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2250,7 +2366,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -2269,9 +2387,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2297,6 +2418,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -2316,6 +2439,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2325,6 +2450,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2349,6 +2475,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2381,6 +2508,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -2398,6 +2526,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2407,6 +2536,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
@@ -2455,7 +2585,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -2505,7 +2637,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -2544,9 +2678,12 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -2592,6 +2729,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -2638,6 +2777,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -2678,6 +2819,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -2714,6 +2857,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -2751,6 +2895,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -2789,6 +2934,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -2828,6 +2974,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
@@ -2868,6 +3015,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
@@ -2920,7 +3068,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -2969,7 +3119,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -3007,9 +3159,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -3054,6 +3209,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3099,6 +3256,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3138,6 +3297,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -3173,6 +3334,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -3209,6 +3371,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -3246,6 +3409,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -3284,6 +3448,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
@@ -3323,6 +3488,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v3
@@ -3401,15 +3567,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3483,15 +3650,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3554,13 +3722,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v7, v8
@@ -3627,13 +3797,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -3702,13 +3873,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -3772,14 +3944,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: buffer_gl1_inv
@@ -3842,13 +4015,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v7, v8
@@ -3910,13 +4084,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v7, v8
@@ -3979,13 +4154,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v7, v8
@@ -4047,13 +4223,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB12_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v6, v4
@@ -4115,13 +4291,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v6, v4
@@ -4186,7 +4363,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
@@ -4230,9 +4409,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v2
@@ -4283,6 +4465,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v2
; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4327,6 +4511,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v2
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4368,6 +4554,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v2
@@ -4410,6 +4597,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v2
@@ -4454,6 +4642,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
@@ -4496,6 +4685,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
@@ -4539,6 +4729,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v2
@@ -4598,7 +4789,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
@@ -4641,9 +4834,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v4
@@ -4693,6 +4889,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v3
; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4736,6 +4934,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
@@ -4776,6 +4976,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
@@ -4817,6 +5018,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v3
@@ -4860,6 +5062,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
@@ -4901,6 +5104,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
@@ -4943,6 +5147,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v1, v3
@@ -5028,15 +5233,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5104,13 +5310,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v7, v4
@@ -5185,13 +5393,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: buffer_gl1_inv
@@ -5260,14 +5469,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
@@ -5335,13 +5545,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB15_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v7, v4
@@ -5408,13 +5619,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v6, v4
@@ -5482,13 +5694,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v6, v4
@@ -5553,13 +5766,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v5, v6
@@ -5625,13 +5838,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_mov_b32_e32 v5, v6
@@ -5678,7 +5892,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v3, v1, v0
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -5718,7 +5934,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v3, v1, v0
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -5753,9 +5971,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -5789,6 +6010,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5826,6 +6049,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5863,6 +6088,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -5895,6 +6122,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -5928,6 +6156,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -5965,6 +6194,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6004,6 +6234,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6047,6 +6278,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -6089,7 +6321,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -6129,7 +6363,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -6163,9 +6399,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -6199,6 +6438,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6236,6 +6477,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6272,6 +6515,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6303,6 +6548,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -6335,6 +6581,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -6371,6 +6618,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -6411,6 +6659,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6451,6 +6700,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -6536,15 +6786,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6625,15 +6876,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6704,13 +6956,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB18_6
; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -6781,13 +7035,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -6860,13 +7115,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -6938,14 +7194,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -7016,13 +7273,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB18_6
; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -7092,13 +7350,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -7172,13 +7431,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -7255,13 +7515,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -7343,13 +7603,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_6
; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -7419,7 +7680,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -7478,7 +7741,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
@@ -7529,9 +7794,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7587,6 +7855,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7644,6 +7914,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7696,6 +7968,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
@@ -7744,6 +8018,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7793,6 +8068,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7843,6 +8119,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
@@ -7887,6 +8164,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -7938,6 +8216,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8003,7 +8282,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -8062,7 +8343,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
@@ -8112,9 +8395,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: v_mov_b32_e32 v3, v4
@@ -8170,6 +8456,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8227,6 +8515,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -8278,6 +8568,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
@@ -8325,6 +8617,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v4
@@ -8373,6 +8666,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v2, v3
@@ -8422,6 +8716,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
@@ -8467,6 +8762,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8514,6 +8810,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8621,15 +8918,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8728,15 +9026,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8824,13 +9123,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_6
; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v9, v6
@@ -8920,13 +9221,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-TRUE16-NEXT: buffer_gl1_inv
@@ -9015,13 +9317,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
; GFX11-FAKE16-NEXT: buffer_gl1_inv
@@ -9107,14 +9410,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
@@ -9201,13 +9505,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_6
; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
@@ -9293,13 +9598,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v8, v6
@@ -9386,13 +9692,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v8, v6
@@ -9473,13 +9780,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -9567,13 +9874,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_6
; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -9609,7 +9917,11 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -9632,8 +9944,11 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc0 sc1
; GFX942-NEXT: buffer_inv sc0 sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -9652,6 +9967,8 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -9663,6 +9980,8 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -9683,10 +10002,14 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: buffer_wbl2
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
+; GFX90A-NEXT: buffer_invl2
+; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -9715,6 +10038,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -9743,6 +10067,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
+; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -9758,6 +10083,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
@@ -9767,6 +10093,7 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
index d2c22b4689dae..4dba78ede565a 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
@@ -17,7 +17,9 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fin
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -44,6 +46,8 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fin
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -67,7 +71,9 @@ define void @buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -92,6 +98,8 @@ define void @buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-NEXT: v_mov_b32_e32 v2, v3
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -116,7 +124,9 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_rem
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -143,6 +153,8 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_rem
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -166,7 +178,9 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fin
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -193,6 +207,8 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fin
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -221,7 +237,11 @@ define i32 @buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fi
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -248,6 +268,8 @@ define i32 @buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fi
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
index 5224090063464..7e5ce9feeaac4 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
@@ -17,7 +17,9 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -31,6 +33,8 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine_grained_memory:
@@ -51,6 +55,7 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
+; GFX9-NEXT: buffer_wbinvl1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -73,7 +78,9 @@ define void @buffer_fat_ptr_agent_atomic_usub_sat_noret_u32__offset__amdgpu_no_f
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -87,6 +94,8 @@ define void @buffer_fat_ptr_agent_atomic_usub_sat_noret_u32__offset__amdgpu_no_f
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: buffer_fat_ptr_agent_atomic_usub_sat_noret_u32__offset__amdgpu_no_fine_grained_memory:
@@ -105,6 +114,7 @@ define void @buffer_fat_ptr_agent_atomic_usub_sat_noret_u32__offset__amdgpu_no_f
; GFX9-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
+; GFX9-NEXT: buffer_wbinvl1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v3
@@ -128,7 +138,9 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_remo
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -142,6 +154,8 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_remo
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_remote_memory:
@@ -162,6 +176,7 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_remo
; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
+; GFX9-NEXT: buffer_wbinvl1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -184,7 +199,9 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -198,6 +215,8 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -218,6 +237,7 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
+; GFX9-NEXT: buffer_wbinvl1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -245,7 +265,11 @@ define i32 @buffer_fat_ptr_system_atomic_usub_sat_ret_u32__offset__amdgpu_no_fin
; GFX12-NEXT: v_mov_b32_e32 v1, s16
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -259,6 +283,8 @@ define i32 @buffer_fat_ptr_system_atomic_usub_sat_ret_u32__offset__amdgpu_no_fin
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: buffer_fat_ptr_system_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine_grained_memory:
@@ -279,6 +305,7 @@ define i32 @buffer_fat_ptr_system_atomic_usub_sat_ret_u32__offset__amdgpu_no_fin
; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
+; GFX9-NEXT: buffer_wbinvl1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
index 9a94a12060f2d..6cb8bdcc473d9 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
@@ -203,7 +203,7 @@ define i32 @fancy_zero() {
define i32 @load_null() {
; CHECK-LABEL: define i32 @load_null() {
-; CHECK-NEXT: [[X:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[X:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: ret i32 [[X]]
;
%x = load i32, ptr addrspace(7) null, align 4
@@ -212,7 +212,7 @@ define i32 @load_null() {
define void @store_null() {
; CHECK-LABEL: define void @store_null() {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
store i32 0, ptr addrspace(7) null, align 4
@@ -221,7 +221,7 @@ define void @store_null() {
define i32 @load_poison() {
; CHECK-LABEL: define i32 @load_poison() {
-; CHECK-NEXT: [[X:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0)
+; CHECK-NEXT: [[X:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret i32 [[X]]
;
%x = load i32, ptr addrspace(7) poison, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
index a221056406da2..7288140c33c13 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
@@ -10,22 +10,22 @@ target triple = "amdgpu9.00--"
define i32 @load_i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define i32 @load_i32_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; STRICT-NEXT: ret i32 [[RET]]
;
; UNALIGNED_ONLY-LABEL: define i32 @load_i32_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; UNALIGNED_ONLY-NEXT: ret i32 [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define i32 @load_i32_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; RELAXED_OOB_ONLY-NEXT: ret i32 [[RET]]
;
; BOTH_FLAGS-LABEL: define i32 @load_i32_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; BOTH_FLAGS-NEXT: ret i32 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -37,22 +37,22 @@ define i32 @load_i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_i32_align4(i32 %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_i32_align4(
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align4(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align4(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align4(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
@@ -65,32 +65,32 @@ define void @store_i32_align4(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
define i32 @load_i32_align2(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define i32 @load_i32_align2(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; STRICT-NEXT: [[RET:%.*]] = bitcast <2 x i16> [[RET_SLICE_1]] to i32
; STRICT-NEXT: ret i32 [[RET]]
;
; UNALIGNED_ONLY-LABEL: define i32 @load_i32_align2(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret i32 [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define i32 @load_i32_align2(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i16> [[RET_SLICE_1]] to i32
; RELAXED_OOB_ONLY-NEXT: ret i32 [[RET]]
;
; BOTH_FLAGS-LABEL: define i32 @load_i32_align2(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret i32 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -139,44 +139,44 @@ define void @store_i32_align2(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
define i32 @load_i32_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define i32 @load_i32_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to i32
; STRICT-NEXT: ret i32 [[RET]]
;
; UNALIGNED_ONLY-LABEL: define i32 @load_i32_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret i32 [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define i32 @load_i32_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to i32
; RELAXED_OOB_ONLY-NEXT: ret i32 [[RET]]
;
; BOTH_FLAGS-LABEL: define i32 @load_i32_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret i32 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -237,22 +237,22 @@ define void @store_i32_align1(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
define i64 @load_i64_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define i64 @load_i64_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret i64 [[RET]]
;
; UNALIGNED_ONLY-LABEL: define i64 @load_i64_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret i64 [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define i64 @load_i64_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret i64 [[RET]]
;
; BOTH_FLAGS-LABEL: define i64 @load_i64_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret i64 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -291,68 +291,68 @@ define void @store_i64_align4(i64 %data, ptr addrspace(8) inreg %buf, i32 %off)
define i64 @load_i64_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define i64 @load_i64_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to i64
; STRICT-NEXT: ret i64 [[RET]]
;
; UNALIGNED_ONLY-LABEL: define i64 @load_i64_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret i64 [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define i64 @load_i64_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add nuw i32 [[Q]], 5
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add nuw i32 [[Q]], 7
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to i64
; RELAXED_OOB_ONLY-NEXT: ret i64 [[RET]]
;
; BOTH_FLAGS-LABEL: define i64 @load_i64_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret i64 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -437,30 +437,30 @@ define void @store_i64_align1(i64 %data, ptr addrspace(8) inreg %buf, i32 %off)
define <2 x i32> @load_v2i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i32> @load_v2i32_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; STRICT-NEXT: ret <2 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i32> @load_v2i32_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i32> @load_v2i32_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <2 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i32> @load_v2i32_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <2 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -507,48 +507,48 @@ define void @store_v2i32_align4(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <2 x i32> @load_v2i32_align2(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i32> @load_v2i32_align2(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i16> poison, i16 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i16> [[RET_SLICE_1]], i16 [[RET_OFF_4]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i16> [[RET_SLICE_2]], i16 [[RET_OFF_6]], i64 3
; STRICT-NEXT: [[RET:%.*]] = bitcast <4 x i16> [[RET_SLICE_3]] to <2 x i32>
; STRICT-NEXT: ret <2 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i32> @load_v2i32_align2(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i32> @load_v2i32_align2(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i16> poison, i16 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i16> [[RET_SLICE_1]], i16 [[RET_OFF_4]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i16> [[RET_SLICE_2]], i16 [[RET_OFF_6]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i16> [[RET_SLICE_3]] to <2 x i32>
; RELAXED_OOB_ONLY-NEXT: ret <2 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i32> @load_v2i32_align2(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <2 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -613,72 +613,72 @@ define void @store_v2i32_align2(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <2 x i32> @load_v2i32_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i32> @load_v2i32_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <2 x i32>
; STRICT-NEXT: ret <2 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i32> @load_v2i32_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i32> @load_v2i32_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add nuw i32 [[Q]], 5
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add nuw i32 [[Q]], 7
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <2 x i32>
; RELAXED_OOB_ONLY-NEXT: ret <2 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i32> @load_v2i32_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <2 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -767,36 +767,36 @@ define void @store_v2i32_align1(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <3 x i32> @load_v3i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <3 x i32> @load_v3i32_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <3 x i32> poison, i32 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <3 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
; STRICT-NEXT: ret <3 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <3 x i32> @load_v3i32_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <3 x i32> poison, i32 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <3 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
; UNALIGNED_ONLY-NEXT: ret <3 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <3 x i32> @load_v3i32_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <3 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <3 x i32> @load_v3i32_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <3 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -849,32 +849,32 @@ define void @store_v3i32_align4(<3 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <3 x i32> @load_v3i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <3 x i32> @load_v3i32_align8(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i32> poison, <3 x i32> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_8]], i64 2
; STRICT-NEXT: ret <3 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <3 x i32> @load_v3i32_align8(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i32> poison, <3 x i32> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_8]], i64 2
; UNALIGNED_ONLY-NEXT: ret <3 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <3 x i32> @load_v3i32_align8(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <3 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <3 x i32> @load_v3i32_align8(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <3 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -921,22 +921,22 @@ define void @store_v3i32_align8(<3 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <3 x i32> @load_v3i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <3 x i32> @load_v3i32_align16(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret <3 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <3 x i32> @load_v3i32_align16(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret <3 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <3 x i32> @load_v3i32_align16(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <3 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <3 x i32> @load_v3i32_align16(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <3 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -975,34 +975,34 @@ define void @store_v3i32_align16(<3 x i32> %data, ptr addrspace(8) inreg %buf, i
define <4 x i32> @load_v4i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <4 x i32> @load_v4i32_align8(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <4 x i32> poison, <4 x i32> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; STRICT-NEXT: [[RET:%.*]] = shufflevector <4 x i32> [[RET_PARTS_0]], <4 x i32> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; STRICT-NEXT: ret <4 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <4 x i32> @load_v4i32_align8(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <4 x i32> poison, <4 x i32> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <4 x i32> [[RET_PARTS_0]], <4 x i32> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; UNALIGNED_ONLY-NEXT: ret <4 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <4 x i32> @load_v4i32_align8(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <4 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <4 x i32> @load_v4i32_align8(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <4 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1049,44 +1049,44 @@ define void @store_v4i32_align8(<4 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <8 x i32> @load_v8i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <8 x i32> @load_v8i32_align16(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; STRICT-NEXT: ret <8 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <8 x i32> @load_v8i32_align16(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; UNALIGNED_ONLY-NEXT: ret <8 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <8 x i32> @load_v8i32_align16(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; RELAXED_OOB_ONLY-NEXT: ret <8 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <8 x i32> @load_v8i32_align16(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; BOTH_FLAGS-NEXT: ret <8 x i32> [[RET]]
@@ -1143,60 +1143,60 @@ define void @store_v8i32_align16(<8 x i32> %data, ptr addrspace(8) inreg %buf, i
define <8 x i32> @load_v8i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <8 x i32> @load_v8i32_align8(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_2]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_2]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
; STRICT-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
-; STRICT-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_4]], <8 x i32> [[RET_EXT_6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; STRICT-NEXT: ret <8 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <8 x i32> @load_v8i32_align8(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_2]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_2]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_4]], <8 x i32> [[RET_EXT_6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: ret <8 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <8 x i32> @load_v8i32_align8(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; RELAXED_OOB_ONLY-NEXT: ret <8 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <8 x i32> @load_v8i32_align8(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; BOTH_FLAGS-NEXT: ret <8 x i32> [[RET]]
@@ -1265,76 +1265,76 @@ define void @store_v8i32_align8(<8 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <8 x i32> @load_v8i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <8 x i32> @load_v8i32_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i32> poison, i32 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_12:%.*]] = add i32 [[Q]], 12
-; STRICT-NEXT: [[RET_OFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i32> [[RET_SLICE_2]], i32 [[RET_OFF_12]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; STRICT-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i32> [[RET_SLICE_3]], i32 [[RET_OFF_16]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_20:%.*]] = add i32 [[Q]], 20
-; STRICT-NEXT: [[RET_OFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_20]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_20]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i32> [[RET_SLICE_4]], i32 [[RET_OFF_20]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
-; STRICT-NEXT: [[RET_OFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i32> [[RET_SLICE_5]], i32 [[RET_OFF_24]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_28:%.*]] = add i32 [[Q]], 28
-; STRICT-NEXT: [[RET_OFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_28]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_28]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <8 x i32> [[RET_SLICE_6]], i32 [[RET_OFF_28]], i64 7
; STRICT-NEXT: ret <8 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <8 x i32> @load_v8i32_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i32> poison, i32 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_12:%.*]] = add i32 [[Q]], 12
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i32> [[RET_SLICE_2]], i32 [[RET_OFF_12]], i64 3
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i32> [[RET_SLICE_3]], i32 [[RET_OFF_16]], i64 4
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_20:%.*]] = add i32 [[Q]], 20
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_20]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_20]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i32> [[RET_SLICE_4]], i32 [[RET_OFF_20]], i64 5
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i32> [[RET_SLICE_5]], i32 [[RET_OFF_24]], i64 6
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_28:%.*]] = add i32 [[Q]], 28
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_28]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_28]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <8 x i32> [[RET_SLICE_6]], i32 [[RET_OFF_28]], i64 7
; UNALIGNED_ONLY-NEXT: ret <8 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <8 x i32> @load_v8i32_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; RELAXED_OOB_ONLY-NEXT: ret <8 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <8 x i32> @load_v8i32_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; BOTH_FLAGS-NEXT: ret <8 x i32> [[RET]]
@@ -1427,60 +1427,60 @@ define void @store_v8i32_align4(<8 x i32> %data, ptr addrspace(8) inreg %buf, i3
define <10 x i32> @load_v10i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <10 x i32> @load_v10i32_align16(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; STRICT-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
-; STRICT-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; STRICT-NEXT: ret <10 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <10 x i32> @load_v10i32_align16(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; UNALIGNED_ONLY-NEXT: ret <10 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <10 x i32> @load_v10i32_align16(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; RELAXED_OOB_ONLY-NEXT: ret <10 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <10 x i32> @load_v10i32_align16(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
-; BOTH_FLAGS-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; BOTH_FLAGS-NEXT: ret <10 x i32> [[RET]]
@@ -1549,76 +1549,76 @@ define void @store_v10i32_align16(<10 x i32> %data, ptr addrspace(8) inreg %buf,
define <10 x i32> @load_v10i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <10 x i32> @load_v10i32_align8(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_2]], <10 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_2]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 6, i32 7, i32 8, i32 9>
; STRICT-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
-; STRICT-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET_PARTS_6:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_6]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
; STRICT-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
-; STRICT-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; STRICT-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_6]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; STRICT-NEXT: ret <10 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <10 x i32> @load_v10i32_align8(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_2]], <10 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_2]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 6, i32 7, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET_PARTS_6:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_6]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_6]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; UNALIGNED_ONLY-NEXT: ret <10 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <10 x i32> @load_v10i32_align8(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; RELAXED_OOB_ONLY-NEXT: ret <10 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <10 x i32> @load_v10i32_align8(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
-; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
-; BOTH_FLAGS-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; BOTH_FLAGS-NEXT: ret <10 x i32> [[RET]]
@@ -1699,32 +1699,32 @@ define void @store_v10i32_align8(<10 x i32> %data, ptr addrspace(8) inreg %buf,
define i16 @load_i16_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define i16 @load_i16_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[RET:%.*]] = bitcast <2 x i8> [[RET_SLICE_1]] to i16
; STRICT-NEXT: ret i16 [[RET]]
;
; UNALIGNED_ONLY-LABEL: define i16 @load_i16_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret i16 [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define i16 @load_i16_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i8> [[RET_SLICE_1]] to i16
; RELAXED_OOB_ONLY-NEXT: ret i16 [[RET]]
;
; BOTH_FLAGS-LABEL: define i16 @load_i16_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret i16 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1773,30 +1773,30 @@ define void @store_i16_align1(i16 %data, ptr addrspace(8) inreg %buf, i32 %off)
define <2 x i16> @load_v2i16_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i16> @load_v2i16_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; STRICT-NEXT: ret <2 x i16> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i16> @load_v2i16_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i16> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i16> @load_v2i16_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <2 x i16> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i16> @load_v2i16_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <2 x i16> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1843,34 +1843,34 @@ define void @store_v2i16_align4(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
define <2 x i16> @load_v2i16_align2(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i16> @load_v2i16_align2(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; STRICT-NEXT: ret <2 x i16> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i16> @load_v2i16_align2(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i16> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i16> @load_v2i16_align2(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; RELAXED_OOB_ONLY-NEXT: ret <2 x i16> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i16> @load_v2i16_align2(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <2 x i16> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1921,48 +1921,48 @@ define void @store_v2i16_align2(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
define <2 x i16> @load_v2i16_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i16> @load_v2i16_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to <2 x i16>
; STRICT-NEXT: ret <2 x i16> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i16> @load_v2i16_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i16> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i16> @load_v2i16_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to <2 x i16>
; RELAXED_OOB_ONLY-NEXT: ret <2 x i16> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i16> @load_v2i16_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <2 x i16> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2027,31 +2027,31 @@ define void @store_v2i16_align1(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
define <2 x i8> @load_v2i8_align2(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i8> @load_v2i8_align2(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: ret <2 x i8> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i8> @load_v2i8_align2(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i8> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i8> @load_v2i8_align2(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
; RELAXED_OOB_ONLY-NEXT: ret <2 x i8> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i8> @load_v2i8_align2(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
; BOTH_FLAGS-NEXT: ret <2 x i8> [[RET]]
;
@@ -2101,34 +2101,34 @@ define void @store_v2i8_align2(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32
define <2 x i8> @load_v2i8_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <2 x i8> @load_v2i8_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: ret <2 x i8> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <2 x i8> @load_v2i8_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: ret <2 x i8> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <2 x i8> @load_v2i8_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: ret <2 x i8> [[RET]]
;
; BOTH_FLAGS-LABEL: define <2 x i8> @load_v2i8_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
; BOTH_FLAGS-NEXT: ret <2 x i8> [[RET]]
;
@@ -2181,43 +2181,43 @@ define void @store_v2i8_align1(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32
define <4 x i8> @load_v4i8_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <4 x i8> @load_v4i8_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: ret <4 x i8> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <4 x i8> @load_v4i8_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: ret <4 x i8> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <4 x i8> @load_v4i8_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; RELAXED_OOB_ONLY-NEXT: ret <4 x i8> [[RET]]
;
; BOTH_FLAGS-LABEL: define <4 x i8> @load_v4i8_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; BOTH_FLAGS-NEXT: ret <4 x i8> [[RET]]
;
@@ -2279,47 +2279,47 @@ define void @store_v4i8_align4(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
define <4 x i8> @load_v4i8_align2(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <4 x i8> @load_v4i8_align2(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: ret <4 x i8> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <4 x i8> @load_v4i8_align2(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: ret <4 x i8> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <4 x i8> @load_v4i8_align2(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i16> [[RET_SLICE_1]] to <4 x i8>
; RELAXED_OOB_ONLY-NEXT: ret <4 x i8> [[RET]]
;
; BOTH_FLAGS-LABEL: define <4 x i8> @load_v4i8_align2(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; BOTH_FLAGS-NEXT: ret <4 x i8> [[RET]]
;
@@ -2385,52 +2385,52 @@ define void @store_v4i8_align2(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
define <4 x i8> @load_v4i8_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <4 x i8> @load_v4i8_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: ret <4 x i8> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <4 x i8> @load_v4i8_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: ret <4 x i8> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <4 x i8> @load_v4i8_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; RELAXED_OOB_ONLY-NEXT: ret <4 x i8> [[RET]]
;
; BOTH_FLAGS-LABEL: define <4 x i8> @load_v4i8_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; BOTH_FLAGS-NEXT: ret <4 x i8> [[RET]]
;
@@ -2501,69 +2501,69 @@ define void @store_v4i8_align1(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
define <16 x i4> @load_v16i4_align8(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <16 x i4> @load_v16i4_align8(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; STRICT-NEXT: ret <16 x i4> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align8(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align8(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align8(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
;
@@ -2651,69 +2651,69 @@ define void @store_v16i4_align8(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
define <16 x i4> @load_v16i4_align4(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <16 x i4> @load_v16i4_align4(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; STRICT-NEXT: ret <16 x i4> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align4(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align4(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align4(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
;
@@ -2801,79 +2801,79 @@ define void @store_v16i4_align4(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
define <16 x i4> @load_v16i4_align2(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <16 x i4> @load_v16i4_align2(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; STRICT-NEXT: ret <16 x i4> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align2(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align2(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i16> poison, i16 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i16> [[RET_SLICE_1]], i16 [[RET_OFF_4]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i16> [[RET_SLICE_2]], i16 [[RET_OFF_6]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i16> [[RET_SLICE_3]] to <16 x i4>
; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align2(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
;
@@ -2971,91 +2971,91 @@ define void @store_v16i4_align2(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
define <16 x i4> @load_v16i4_align1(ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define <16 x i4> @load_v16i4_align1(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; STRICT-NEXT: ret <16 x i4> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align1(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
-; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align1(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add nuw i32 [[Q]], 5
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add nuw i32 [[Q]], 7
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
;
; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align1(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
;
@@ -3166,25 +3166,25 @@ define <4 x i32> @load_v4i32_align8_nneg(ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) {
; STRICT-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
-; STRICT-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret <4 x i32> [[RET]]
;
; UNALIGNED_ONLY-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret <4 x i32> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
-; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret <4 x i32> [[RET]]
;
; BOTH_FLAGS-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <4 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3199,28 +3199,28 @@ define <8 x half> @load_v8f16_align2_not_oob(ptr addrspace(1) inreg %ptr, i32 %o
; STRICT-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) {
; STRICT-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
; STRICT-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
-; STRICT-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x half> poison, half [[RET_OFF_0]], i64 0
; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
-; STRICT-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
-; STRICT-NEXT: [[RET_OFF_4:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x half> [[RET_SLICE_1]], half [[RET_OFF_4]], i64 2
; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
-; STRICT-NEXT: [[RET_OFF_6:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x half> [[RET_SLICE_2]], half [[RET_OFF_6]], i64 3
; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
-; STRICT-NEXT: [[RET_OFF_8:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x half> [[RET_SLICE_3]], half [[RET_OFF_8]], i64 4
; STRICT-NEXT: [[Q_OFF_PTR_10:%.*]] = add i32 [[Q]], 10
-; STRICT-NEXT: [[RET_OFF_10:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_10]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_10:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_10]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x half> [[RET_SLICE_4]], half [[RET_OFF_10]], i64 5
; STRICT-NEXT: [[Q_OFF_PTR_12:%.*]] = add i32 [[Q]], 12
-; STRICT-NEXT: [[RET_OFF_12:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_12:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x half> [[RET_SLICE_5]], half [[RET_OFF_12]], i64 6
; STRICT-NEXT: [[Q_OFF_PTR_14:%.*]] = add i32 [[Q]], 14
-; STRICT-NEXT: [[RET_OFF_14:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_14]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_OFF_14:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_14]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[RET:%.*]] = insertelement <8 x half> [[RET_SLICE_6]], half [[RET_OFF_14]], i64 7
; STRICT-NEXT: ret <8 x half> [[RET]]
;
@@ -3228,35 +3228,35 @@ define <8 x half> @load_v8f16_align2_not_oob(ptr addrspace(1) inreg %ptr, i32 %o
; UNALIGNED_ONLY-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
; UNALIGNED_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
-; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret <8 x half> [[RET]]
;
; RELAXED_OOB_ONLY-LABEL: define <8 x half> @load_v8f16_align2_not_oob(
; RELAXED_OOB_ONLY-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
; RELAXED_OOB_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x half> poison, half [[RET_OFF_0]], i64 0
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x half> [[RET_SLICE_1]], half [[RET_OFF_4]], i64 2
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x half> [[RET_SLICE_2]], half [[RET_OFF_6]], i64 3
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add nuw i32 [[Q]], 8
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_8:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_8:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x half> [[RET_SLICE_3]], half [[RET_OFF_8]], i64 4
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_10:%.*]] = add nuw i32 [[Q]], 10
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_10:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_10]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_10:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_10]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x half> [[RET_SLICE_4]], half [[RET_OFF_10]], i64 5
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_12:%.*]] = add nuw i32 [[Q]], 12
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_12:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_12:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x half> [[RET_SLICE_5]], half [[RET_OFF_12]], i64 6
; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_14:%.*]] = add nuw i32 [[Q]], 14
-; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_14:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_14]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_14:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_14]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <8 x half> [[RET_SLICE_6]], half [[RET_OFF_14]], i64 7
; RELAXED_OOB_ONLY-NEXT: ret <8 x half> [[RET]]
;
@@ -3264,7 +3264,7 @@ define <8 x half> @load_v8f16_align2_not_oob(ptr addrspace(1) inreg %ptr, i32 %o
; BOTH_FLAGS-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
; BOTH_FLAGS-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
-; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret <8 x half> [[RET]]
;
%buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %ptr, i16 0, i64 8192, i32 159744)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
index 03fef762d8818..54421cac3a033 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
@@ -12,7 +12,7 @@ target triple = "amdgpu9.00--"
define i8 @load_i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i8 @load_i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1:![0-9]+]])
; CHECK-NEXT: ret i8 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -23,7 +23,7 @@ define i8 @load_i8(ptr addrspace(8) inreg %buf) {
define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i8(
; CHECK-SAME: i8 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1:![0-9]+]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -34,7 +34,7 @@ define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {
define i16 @load_i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i16 @load_i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret i16 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -56,7 +56,7 @@ define void @store_i16(i16 %data, ptr addrspace(8) inreg %buf) {
define i32 @load_i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i32 @load_i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret i32 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -78,7 +78,7 @@ define void @store_i32(i32 %data, ptr addrspace(8) inreg %buf) {
define i64 @load_i64(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i64 @load_i64(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret i64 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -100,7 +100,7 @@ define void @store_i64(i64 %data, ptr addrspace(8) inreg %buf) {
define i128 @load_i128(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i128 @load_i128(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call i128 @llvm.amdgcn.raw.ptr.buffer.load.i128(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i128 @llvm.amdgcn.raw.ptr.buffer.load.i128(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret i128 [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -122,7 +122,7 @@ define void @store_i128(i128 %data, ptr addrspace(8) inreg %buf) {
define <1 x i32> @load_v1i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <1 x i32> @load_v1i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <1 x i32>
; CHECK-NEXT: ret <1 x i32> [[RET]]
;
@@ -146,7 +146,7 @@ define void @store_v1i32(<1 x i32> %data, ptr addrspace(8) inreg %buf) {
define <2 x i32> @load_v2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i32> @load_v2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -168,7 +168,7 @@ define void @store_v2i32(<2 x i32> %data, ptr addrspace(8) inreg %buf) {
define <3 x i32> @load_v3i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x i32> @load_v3i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -190,7 +190,7 @@ define void @store_v3i32(<3 x i32> %data, ptr addrspace(8) inreg %buf) {
define <4 x i32> @load_v4i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i32> @load_v4i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -212,7 +212,7 @@ define void @store_v4i32(<4 x i32> %data, ptr addrspace(8) inreg %buf) {
define <2 x i16> @load_v2i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i16> @load_v2i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i16> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -234,7 +234,7 @@ define void @store_v2i16(<2 x i16> %data, ptr addrspace(8) inreg %buf) {
define <4 x i16> @load_v4i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i16> @load_v4i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x i16> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -256,7 +256,7 @@ define void @store_v4i16(<4 x i16> %data, ptr addrspace(8) inreg %buf) {
define <8 x i16> @load_v8i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i16> @load_v8i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <8 x i16> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -278,7 +278,7 @@ define void @store_v8i16(<8 x i16> %data, ptr addrspace(8) inreg %buf) {
define <2 x i64> @load_v2i64(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i64> @load_v2i64(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x i64> @llvm.amdgcn.raw.ptr.buffer.load.v2i64(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x i64> @llvm.amdgcn.raw.ptr.buffer.load.v2i64(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i64> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -300,7 +300,7 @@ define void @store_v2i64(<2 x i64> %data, ptr addrspace(8) inreg %buf) {
define half @load_f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define half @load_f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -322,7 +322,7 @@ define void @store_f16(half %data, ptr addrspace(8) inreg %buf) {
define bfloat @load_bf16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define bfloat @load_bf16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call bfloat @llvm.amdgcn.raw.ptr.buffer.load.bf16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call bfloat @llvm.amdgcn.raw.ptr.buffer.load.bf16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret bfloat [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -344,7 +344,7 @@ define void @store_bf16(bfloat %data, ptr addrspace(8) inreg %buf) {
define <2 x half> @load_v2f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x half> @load_v2f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x half> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -366,9 +366,9 @@ define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {
define <2 x half> @load_v2f16_align2(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x half> @load_v2f16_align2(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x half> poison, half [[RET_OFF_0]], i64 0
-; CHECK-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <2 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
; CHECK-NEXT: ret <2 x half> [[RET]]
;
@@ -394,7 +394,7 @@ define void @store_v2f16_align2(<2 x half> %data, ptr addrspace(8) inreg %buf) {
define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x bfloat> @load_v4bf16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <4 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v4bf16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <4 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v4bf16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x bfloat> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -416,7 +416,7 @@ define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {
define <8 x half> @load_v8f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x half> @load_v8f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <8 x half> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -438,7 +438,7 @@ define void @store_v8f16(<8 x half> %data, ptr addrspace(8) inreg %buf) {
define float @load_f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define float @load_f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -460,7 +460,7 @@ define void @store_f32(float %data, ptr addrspace(8) inreg %buf) {
define <2 x float> @load_v2f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x float> @load_v2f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -482,7 +482,7 @@ define void @store_v2f32(<2 x float> %data, ptr addrspace(8) inreg %buf) {
define <3 x float> @load_v3f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x float> @load_v3f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -504,7 +504,7 @@ define void @store_v3f32(<3 x float> %data, ptr addrspace(8) inreg %buf) {
define <4 x float> @load_v4f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x float> @load_v4f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -526,7 +526,7 @@ define void @store_v4f32(<4 x float> %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(0) @load_p0(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr @load_p0(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -548,7 +548,7 @@ define void @store_p0(ptr addrspace(0) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(1) @load_p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(1) @load_p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(1) @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(1) @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(1) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -570,7 +570,7 @@ define void @store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(2) @load_p2(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(2) @load_p2(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(2) @llvm.amdgcn.raw.ptr.buffer.load.p2(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(2) @llvm.amdgcn.raw.ptr.buffer.load.p2(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(2) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -592,7 +592,7 @@ define void @store_p2(ptr addrspace(2) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(3) @load_p3(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(3) @load_p3(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(3) @llvm.amdgcn.raw.ptr.buffer.load.p3(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(3) @llvm.amdgcn.raw.ptr.buffer.load.p3(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(3) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -614,7 +614,7 @@ define void @store_p3(ptr addrspace(3) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(4) @load_p4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(4) @load_p4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(4) @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(4) @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(4) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -636,7 +636,7 @@ define void @store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(5) @load_p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(5) @load_p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(5) @llvm.amdgcn.raw.ptr.buffer.load.p5(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(5) @llvm.amdgcn.raw.ptr.buffer.load.p5(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(5) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -658,7 +658,7 @@ define void @store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(6) @load_p6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(6) @load_p6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(6) @llvm.amdgcn.raw.ptr.buffer.load.p6(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(6) @llvm.amdgcn.raw.ptr.buffer.load.p6(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(6) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -680,7 +680,7 @@ define void @store_p6(ptr addrspace(6) %data, ptr addrspace(8) inreg %buf) {
define ptr addrspace(8) @load_p8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(8) @load_p8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(8) @llvm.amdgcn.raw.ptr.buffer.load.p8(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call ptr addrspace(8) @llvm.amdgcn.raw.ptr.buffer.load.p8(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret ptr addrspace(8) [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -702,7 +702,7 @@ define void @store_p8(ptr addrspace(8) %data, ptr addrspace(8) inreg %buf) {
define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x ptr addrspace(1)> @load_v2p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x ptr addrspace(1)> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -724,7 +724,7 @@ define void @store_v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %bu
define <2 x ptr addrspace(5)> @load_v2p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x ptr addrspace(5)> @load_v2p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <2 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v2p5(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <2 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v2p5(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x ptr addrspace(5)> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -746,7 +746,7 @@ define void @store_v2p5(<2 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %bu
define <3 x ptr addrspace(5)> @load_v3p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x ptr addrspace(5)> @load_v3p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <3 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v3p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <3 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v3p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x ptr addrspace(5)> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -768,7 +768,7 @@ define void @store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %bu
define <4 x ptr addrspace(5)> @load_v4p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x ptr addrspace(5)> @load_v4p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <4 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v4p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <4 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v4p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x ptr addrspace(5)> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -792,7 +792,7 @@ define void @store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %bu
define <6 x half> @load_v6f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x half> @load_v6f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <6 x half>
; CHECK-NEXT: ret <6 x half> [[RET]]
;
@@ -818,10 +818,10 @@ define void @store_v6f16(<6 x half> %data, ptr addrspace(8) inreg %buf) {
define <5 x float> @load_v5f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <5 x float> @load_v5f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <5 x float> poison, <5 x float> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <5 x float> [[RET_PARTS_0]], float [[RET_OFF_16]], i64 4
; CHECK-NEXT: ret <5 x float> [[RET]]
;
@@ -847,10 +847,10 @@ define void @store_v5f32(<5 x float> %data, ptr addrspace(8) inreg %buf) {
define <6 x float> @load_v6f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x float> @load_v6f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <6 x float> poison, <6 x float> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x float> [[RET_OFF_16]], <2 x float> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET:%.*]] = shufflevector <6 x float> [[RET_PARTS_0]], <6 x float> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT: ret <6 x float> [[RET]]
@@ -877,10 +877,10 @@ define void @store_v6f32(<6 x float> %data, ptr addrspace(8) inreg %buf) {
define <7 x float> @load_v7f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <7 x float> @load_v7f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <7 x float> poison, <7 x float> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 4, i32 5, i32 6>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <3 x float> [[RET_OFF_16]], <3 x float> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET:%.*]] = shufflevector <7 x float> [[RET_PARTS_0]], <7 x float> [[RET_EXT_4]], <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 8, i32 9>
; CHECK-NEXT: ret <7 x float> [[RET]]
@@ -907,10 +907,10 @@ define void @store_v7f32(<7 x float> %data, ptr addrspace(8) inreg %buf) {
define <8 x float> @load_v8f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x float> @load_v8f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x float> poison, <8 x float> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x float> [[RET_OFF_16]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET:%.*]] = shufflevector <8 x float> [[RET_PARTS_0]], <8 x float> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: ret <8 x float> [[RET]]
@@ -937,13 +937,13 @@ define void @store_v8f32(<8 x float> %data, ptr addrspace(8) inreg %buf) {
define <10 x float> @load_v10f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <10 x float> @load_v10f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x float> poison, <10 x float> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x float> [[RET_OFF_16]], <4 x float> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x float> [[RET_PARTS_0]], <10 x float> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
-; CHECK-NEXT: [[RET_OFF_32:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_32:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x float> [[RET_OFF_32]], <2 x float> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET:%.*]] = shufflevector <10 x float> [[RET_PARTS_4]], <10 x float> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; CHECK-NEXT: ret <10 x float> [[RET]]
@@ -972,10 +972,10 @@ define void @store_v10f32(<10 x float> %data, ptr addrspace(8) inreg %buf) {
define <6 x i32> @load_v6i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i32> @load_v6i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT: ret <6 x i32> [[RET]]
@@ -1002,10 +1002,10 @@ define void @store_v6i32(<6 x i32> %data, ptr addrspace(8) inreg %buf) {
define <4 x ptr addrspace(1)> @load_v4p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x ptr addrspace(1)> @load_v4p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x ptr addrspace(1)> [[RET_OFF_0]], <2 x ptr addrspace(1)> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <4 x ptr addrspace(1)> poison, <4 x ptr addrspace(1)> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x ptr addrspace(1)> [[RET_OFF_16]], <2 x ptr addrspace(1)> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[RET:%.*]] = shufflevector <4 x ptr addrspace(1)> [[RET_PARTS_0]], <4 x ptr addrspace(1)> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: ret <4 x ptr addrspace(1)> [[RET]]
@@ -1034,7 +1034,7 @@ define void @store_v4p1(<4 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %bu
define <1 x i16> @load_v1i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <1 x i16> @load_v1i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <1 x i16>
; CHECK-NEXT: ret <1 x i16> [[RET]]
;
@@ -1058,10 +1058,10 @@ define void @store_v1i16(<1 x i16> %data, ptr addrspace(8) inreg %buf) {
define <3 x i16> @load_v3i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x i16> @load_v3i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 2
; CHECK-NEXT: ret <3 x i16> [[RET]]
;
@@ -1087,10 +1087,10 @@ define void @store_v3i16(<3 x i16> %data, ptr addrspace(8) inreg %buf) {
define <5 x i16> @load_v5i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <5 x i16> @load_v5i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i16> [[RET_OFF_0]], <4 x i16> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <5 x i16> poison, <5 x i16> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
-; CHECK-NEXT: [[RET_OFF_8:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_8:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <5 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_8]], i64 4
; CHECK-NEXT: ret <5 x i16> [[RET]]
;
@@ -1116,7 +1116,7 @@ define void @store_v5i16(<5 x i16> %data, ptr addrspace(8) inreg %buf) {
define <6 x i16> @load_v6i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i16> @load_v6i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <6 x i16>
; CHECK-NEXT: ret <6 x i16> [[RET]]
;
@@ -1140,11 +1140,11 @@ define void @store_v6i16(<6 x i16> %data, ptr addrspace(8) inreg %buf) {
define <7 x i16> @load_v7i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <7 x i16> @load_v7i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast <3 x i32> [[RET_OFF_0]] to <6 x i16>
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <6 x i16> [[RET_OFF_0_FROM_LOADABLE]], <6 x i16> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <7 x i16> poison, <7 x i16> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 6>
-; CHECK-NEXT: [[RET_OFF_12:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_12:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <7 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_12]], i64 6
; CHECK-NEXT: ret <7 x i16> [[RET]]
;
@@ -1171,10 +1171,10 @@ define void @store_v7i16(<7 x i16> %data, ptr addrspace(8) inreg %buf) {
define <9 x i16> @load_v9i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <9 x i16> @load_v9i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <8 x i16> [[RET_OFF_0]], <8 x i16> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <9 x i16> poison, <9 x i16> [[RET_EXT_0]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <9 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_16]], i64 8
; CHECK-NEXT: ret <9 x i16> [[RET]]
;
@@ -1204,7 +1204,7 @@ define void @store_v9i16(<9 x i16> %data, ptr addrspace(8) inreg %buf) {
define <1 x i8> @load_v1i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <1 x i8> @load_v1i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i8 [[RET_LOADABLE]] to <1 x i8>
; CHECK-NEXT: ret <1 x i8> [[RET]]
;
@@ -1228,7 +1228,7 @@ define void @store_v1i8(<1 x i8> %data, ptr addrspace(8) inreg %buf) {
define <2 x i8> @load_v2i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i8> @load_v2i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
; CHECK-NEXT: ret <2 x i8> [[RET]]
;
@@ -1252,11 +1252,11 @@ define void @store_v2i8(<2 x i8> %data, ptr addrspace(8) inreg %buf) {
define <3 x i8> @load_v3i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x i8> @load_v3i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i16 [[RET_OFF_0]] to <2 x i8>
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i8> [[RET_OFF_0_FROM_LOADABLE]], <2 x i8> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i8> poison, <3 x i8> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
-; CHECK-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <3 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_2]], i64 2
; CHECK-NEXT: ret <3 x i8> [[RET]]
;
@@ -1283,7 +1283,7 @@ define void @store_v3i8(<3 x i8> %data, ptr addrspace(8) inreg %buf) {
define <4 x i8> @load_v4i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i8> @load_v4i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; CHECK-NEXT: ret <4 x i8> [[RET]]
;
@@ -1307,11 +1307,11 @@ define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
define <5 x i8> @load_v5i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <5 x i8> @load_v5i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <5 x i8> poison, <5 x i8> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <5 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_4]], i64 4
; CHECK-NEXT: ret <5 x i8> [[RET]]
;
@@ -1338,10 +1338,10 @@ define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {
define <6 x i8> @load_v6i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i8> @load_v6i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_2:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 2
; CHECK-NEXT: [[RET:%.*]] = bitcast <3 x i16> [[RET_SLICE_2]] to <6 x i8>
; CHECK-NEXT: ret <6 x i8> [[RET]]
@@ -1369,15 +1369,15 @@ define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
define <7 x i8> @load_v7i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <7 x i8> @load_v7i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <7 x i8> poison, <7 x i8> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 4, i32 5, i32 6>
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_4_FROM_LOADABLE:%.*]] = bitcast i16 [[RET_OFF_4]] to <2 x i8>
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i8> [[RET_OFF_4_FROM_LOADABLE]], <2 x i8> poison, <7 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <7 x i8> [[RET_PARTS_0]], <7 x i8> [[RET_EXT_4]], <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 8, i32 6>
-; CHECK-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <7 x i8> [[RET_PARTS_4]], i8 [[RET_OFF_6]], i64 6
; CHECK-NEXT: ret <7 x i8> [[RET]]
;
@@ -1407,7 +1407,7 @@ define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {
define <8 x i8> @load_v8i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i8> @load_v8i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <8 x i8>
; CHECK-NEXT: ret <8 x i8> [[RET]]
;
@@ -1431,21 +1431,21 @@ define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
define <8 x i8> @load_v8i8_align1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i8> @load_v8i8_align1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
-; CHECK-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
-; CHECK-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
-; CHECK-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
-; CHECK-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
-; CHECK-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
-; CHECK-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
; CHECK-NEXT: ret <8 x i8> [[RET]]
;
@@ -1483,7 +1483,7 @@ define void @store_v8i8_align1(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
define <12 x i8> @load_v12i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <12 x i8> @load_v12i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <12 x i8>
; CHECK-NEXT: ret <12 x i8> [[RET]]
;
@@ -1507,7 +1507,7 @@ define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {
define <16 x i8> @load_v16i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <16 x i8> @load_v16i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast <4 x i32> [[RET_LOADABLE]] to <16 x i8>
; CHECK-NEXT: ret <16 x i8> [[RET]]
;
@@ -1531,10 +1531,10 @@ define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {
define <32 x i8> @load_v32i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <32 x i8> @load_v32i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: [[RET:%.*]] = bitcast <8 x i32> [[RET_PARTS_4]] to <32 x i8>
@@ -1565,7 +1565,7 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
define [1 x i32] @load_a1i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [1 x i32] @load_a1i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_LOADABLE]] to <1 x i32>
; CHECK-NEXT: [[RET_ELEM_0:%.*]] = extractelement <1 x i32> [[RET_FROM_LOADABLE]], i64 0
; CHECK-NEXT: [[RET:%.*]] = insertvalue [1 x i32] poison, i32 [[RET_ELEM_0]], 0
@@ -1593,7 +1593,7 @@ define void @store_a1i32([1 x i32] %data, ptr addrspace(8) inreg %buf) {
define [2 x i32] @load_a2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x i32] @load_a2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 0
; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET_OFF_0]], 0
; CHECK-NEXT: [[RET_OFF_4:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 1
@@ -1623,9 +1623,9 @@ define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {
define [2 x half] @load_a2f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x half] @load_a2f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x half> poison, half [[RET_OFF_0]], i64 0
-; CHECK-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_LOADABLE:%.*]] = insertelement <2 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x half] poison, half [[RET_OFF_0]], 0
; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x half] [[RET_AS_ARRAY_0]], half [[RET_OFF_2]], 1
@@ -1655,7 +1655,7 @@ define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {
define [2 x ptr addrspace(1)] @load_a2p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x ptr addrspace(1)] @load_a2p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 0
; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x ptr addrspace(1)] poison, ptr addrspace(1) [[RET_OFF_0]], 0
; CHECK-NEXT: [[RET_OFF_8:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 1
@@ -1687,11 +1687,11 @@ define void @store_a2p1([2 x ptr addrspace(1)] %data, ptr addrspace(8) inreg %bu
define i40 @load_i40(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i40 @load_i40(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <5 x i8> poison, <5 x i8> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_4:%.*]] = insertelement <5 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_4]], i64 4
; CHECK-NEXT: [[RET:%.*]] = bitcast <5 x i8> [[RET_SLICE_4]] to i40
; CHECK-NEXT: ret i40 [[RET]]
@@ -1720,7 +1720,7 @@ define void @store_i40(i40 %data, ptr addrspace(8) inreg %buf) {
define i96 @load_i96(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i96 @load_i96(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to i96
; CHECK-NEXT: ret i96 [[RET]]
;
@@ -1744,10 +1744,10 @@ define void @store_i96(i96 %data, ptr addrspace(8) inreg %buf) {
define i160 @load_i160(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i160 @load_i160(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <5 x i32> poison, <5 x i32> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_4:%.*]] = insertelement <5 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_16]], i64 4
; CHECK-NEXT: [[RET:%.*]] = bitcast <5 x i32> [[RET_SLICE_4]] to i160
; CHECK-NEXT: ret i160 [[RET]]
@@ -1775,10 +1775,10 @@ define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {
define i256 @load_i256(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i256 @load_i256(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: [[RET:%.*]] = bitcast <8 x i32> [[RET_PARTS_4]] to i256
@@ -1809,7 +1809,7 @@ define void @store_i256(i256 %data, ptr addrspace(8) inreg %buf) {
define i7 @load_i7(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i7 @load_i7(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = trunc i8 [[RET_LOADABLE]] to i7
; CHECK-NEXT: ret i7 [[RET]]
;
@@ -1833,7 +1833,7 @@ define void @store_i7(i7 %data, ptr addrspace(8) inreg %buf) {
define i4 @load_i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i4 @load_i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = trunc i8 [[RET_LOADABLE]] to i4
; CHECK-NEXT: ret i4 [[RET]]
;
@@ -1859,7 +1859,7 @@ define void @store_i4(i4 %data, ptr addrspace(8) inreg %buf) {
define <2 x i4> @load_v2i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i4> @load_v2i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i8 [[RET_LOADABLE]] to <2 x i4>
; CHECK-NEXT: ret <2 x i4> [[RET]]
;
@@ -1883,7 +1883,7 @@ define void @store_v2i4(<2 x i4> %data, ptr addrspace(8) inreg %buf) {
define <4 x i4> @load_v4i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i4> @load_v4i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <4 x i4>
; CHECK-NEXT: ret <4 x i4> [[RET]]
;
@@ -1907,7 +1907,7 @@ define void @store_v4i4(<4 x i4> %data, ptr addrspace(8) inreg %buf) {
define <8 x i4> @load_v8i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i4> @load_v8i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <8 x i4>
; CHECK-NEXT: ret <8 x i4> [[RET]]
;
@@ -1933,7 +1933,7 @@ define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {
define <2 x i6> @load_v2i6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i6> @load_v2i6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_TRUNC:%.*]] = trunc i16 [[RET_LOADABLE]] to i12
; CHECK-NEXT: [[RET:%.*]] = bitcast i12 [[RET_TRUNC]] to <2 x i6>
; CHECK-NEXT: ret <2 x i6> [[RET]]
@@ -1960,10 +1960,10 @@ define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {
define <6 x i32> @load_v32i6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i32> @load_v32i6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT: [[RET:%.*]] = bitcast <6 x i32> [[RET_PARTS_4]] to <32 x i6>
@@ -1997,7 +1997,7 @@ define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {
define <4 x i8> @volatile_load_v4i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i8> @volatile_load_v4i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; CHECK-NEXT: ret <4 x i8> [[RET]]
;
@@ -2021,10 +2021,10 @@ define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
define <6 x i8> @volatile_load_v6i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i8> @volatile_load_v6i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
-; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: [[RET_SLICE_2:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 2
; CHECK-NEXT: [[RET:%.*]] = bitcast <3 x i16> [[RET_SLICE_2]] to <6 x i8>
; CHECK-NEXT: ret <6 x i8> [[RET]]
@@ -2052,13 +2052,13 @@ define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
define [2 x [2 x i32]] @load_a2a2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x [2 x i32]] @load_a2a2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET0_OFF_1:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 0
; CHECK-NEXT: [[RET0_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET0_OFF_1]], 0
; CHECK-NEXT: [[RET0_OFF_4:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 1
; CHECK-NEXT: [[RET0_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET0_AS_ARRAY_0]], i32 [[RET0_OFF_4]], 1
; CHECK-NEXT: [[RET0:%.*]] = insertvalue [2 x [2 x i32]] poison, [2 x i32] [[RET0_AS_ARRAY_1]], 0
-; CHECK-NEXT: [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET1_OFF_9:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 0
; CHECK-NEXT: [[RET1_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET1_OFF_9]], 0
; CHECK-NEXT: [[RET1_OFF_12:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 1
@@ -2096,9 +2096,9 @@ define void @store_a2a2i32([2 x [2 x i32]] %data, ptr addrspace(8) inreg %buf) {
define [2 x <2 x i32>] @load_a2v2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x <2 x i32>] @load_a2v2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET0:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[RET0_OFF_0]], 0
-; CHECK-NEXT: [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x <2 x i32>] [[RET0]], <2 x i32> [[RET1_OFF_8]], 1
; CHECK-NEXT: ret [2 x <2 x i32>] [[RET]]
;
@@ -2124,9 +2124,9 @@ define void @store_a2v2i32([2 x <2 x i32>] %data, ptr addrspace(8) inreg %buf) {
define [2 x <3 x i32>] @load_a2v3i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x <3 x i32>] @load_a2v3i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET0:%.*]] = insertvalue [2 x <3 x i32>] poison, <3 x i32> [[RET0_OFF_0]], 0
-; CHECK-NEXT: [[RET1_OFF_16:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET1_OFF_16:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x <3 x i32>] [[RET0]], <3 x i32> [[RET1_OFF_16]], 1
; CHECK-NEXT: ret [2 x <3 x i32>] [[RET]]
;
@@ -2152,7 +2152,7 @@ define void @store_a2v3i32([2 x <3 x i32>] %data, ptr addrspace(8) inreg %buf) {
define { i32 } @load_sl_i32s(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { i32 } @load_sl_i32s(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_0_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_0_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertvalue { i32 } poison, i32 [[RET_0_OFF_0]], 0
; CHECK-NEXT: ret { i32 } [[RET]]
;
@@ -2176,7 +2176,7 @@ define void @store_sl_i32s({ i32 } %data, ptr addrspace(8) inreg %buf) {
define { { float } } @load_sl_sl_f32ss(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { { float } } @load_sl_sl_f32ss(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_0_0_OFF_0:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_0_0_OFF_0:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertvalue { { float } } poison, float [[RET_0_0_OFF_0]], 0, 0
; CHECK-NEXT: ret { { float } } [[RET]]
;
@@ -2200,7 +2200,7 @@ define void @store_sl_sl_f32ss({ { float } } %data, ptr addrspace(8) inreg %buf)
define { <2 x i32> } @load_sl_v2i32s(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { <2 x i32> } @load_sl_v2i32s(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertvalue { <2 x i32> } poison, <2 x i32> [[RET_0_OFF_0]], 0
; CHECK-NEXT: ret { <2 x i32> } [[RET]]
;
@@ -2224,9 +2224,9 @@ define void @store_sl_v2i32s({ <2 x i32> } %data, ptr addrspace(8) inreg %buf) {
define { i64, i32 } @load_sl_i64i32s(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { i64, i32 } @load_sl_i64i32s(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET_0_OFF_0:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_0_OFF_0:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET_0:%.*]] = insertvalue { i64, i32 } poison, i64 [[RET_0_OFF_0]], 0
-; CHECK-NEXT: [[RET_1_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: [[RET_1_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET:%.*]] = insertvalue { i64, i32 } [[RET_0]], i32 [[RET_1_OFF_8]], 1
; CHECK-NEXT: ret { i64, i32 } [[RET]]
;
@@ -2252,16 +2252,16 @@ define void @store_sl_i64i32s({ i64, i32 } %data, ptr addrspace(8) inreg %buf) {
define [4 x i7] @load_a4i7(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [4 x i7] @load_a4i7(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET0_TRUNC:%.*]] = trunc i8 [[RET0_OFF_0]] to i7
; CHECK-NEXT: [[RET0:%.*]] = insertvalue [4 x i7] poison, i7 [[RET0_TRUNC]], 0
-; CHECK-NEXT: [[RET1_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: [[RET1_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET1_TRUNC:%.*]] = trunc i8 [[RET1_OFF_1]] to i7
; CHECK-NEXT: [[RET1:%.*]] = insertvalue [4 x i7] [[RET0]], i7 [[RET1_TRUNC]], 1
-; CHECK-NEXT: [[RET2_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET2_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET2_TRUNC:%.*]] = trunc i8 [[RET2_OFF_2]] to i7
; CHECK-NEXT: [[RET2:%.*]] = insertvalue [4 x i7] [[RET1]], i7 [[RET2_TRUNC]], 2
-; CHECK-NEXT: [[RET3_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: [[RET3_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[RET3_TRUNC:%.*]] = trunc i8 [[RET3_OFF_3]] to i7
; CHECK-NEXT: [[RET:%.*]] = insertvalue [4 x i7] [[RET2]], i7 [[RET3_TRUNC]], 3
; CHECK-NEXT: ret [4 x i7] [[RET]]
@@ -2298,7 +2298,7 @@ define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {
define <vscale x 2 x i32> @load_nxv2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <vscale x 2 x i32> @load_nxv2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: [[RET:%.*]] = call <vscale x 2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.nxv2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call <vscale x 2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.nxv2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret <vscale x 2 x i32> [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll
index b7dda33b6646c..d7ba3bf4efeec 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll
@@ -14,7 +14,7 @@ define float @sum(ptr addrspace(8) %buf, i32 %len) {
; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[PTR]] = add i32 [[PTR_PREV_OFF]], 4
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
@@ -53,7 +53,7 @@ define float @sum_integer_ops(ptr addrspace(8) %buf, i32 %len) {
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP]] ], [ [[BUF]], [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[PTR_PREV_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_PREV_RSRC]] to i160
; CHECK-NEXT: [[TMP0:%.*]] = shl nuw i160 [[PTR_PREV_INT_RSRC]], 32
@@ -106,7 +106,7 @@ define float @sum_2d(ptr addrspace(8) %buf, i32 %ii, i32 %jj) {
; CHECK-NEXT: [[SUM2_PREV:%.*]] = phi float [ [[SUM]], [[LOOP2]] ], [ [[SUM1_PREV]], [[LOOP1_ENTRY]] ]
; CHECK-NEXT: [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[LOOP2]] ], [ 0, [[LOOP1_ENTRY]] ]
; CHECK-NEXT: [[PTR2_PREV_OFF:%.*]] = phi i32 [ [[PTR2:%.*]], [[LOOP2]] ], [ [[PTR1_PREV_OFF]], [[LOOP1_ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR2_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR2_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM2_PREV]], [[VAL]]
; CHECK-NEXT: [[PTR2]] = add i32 [[PTR2_PREV_OFF]], 4
; CHECK-NEXT: [[J_NEXT]] = add i32 [[J]], 1
@@ -162,7 +162,7 @@ define float @sum_jump_on_negative(ptr addrspace(8) %buf, i32 %len) {
; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[SKIP_NEXT:%.*]] = fcmp olt float [[VAL]], 0.000000e+00
; CHECK-NEXT: [[SMALL_JUMP:%.*]] = add i32 [[PTR_PREV_OFF]], 4
@@ -206,7 +206,7 @@ define float @sum_jump_on_negative_with_phi(ptr addrspace(8) %buf, i32 %len) {
; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP_EXIT:%.*]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]]
@@ -264,7 +264,7 @@ define float @sum_new_buffer_on_negative(ptr addrspace(8) %buf1, ptr addrspace(8
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP]] ], [ [[BUF1]], [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[HOP:%.*]] = fcmp olt float [[VAL]], 0.000000e+00
; CHECK-NEXT: [[THIS_NEXT:%.*]] = add i32 [[PTR_PREV_OFF]], 4
@@ -310,7 +310,7 @@ define float @sum_new_buffer_on_negative_with_phi(ptr addrspace(8) %buf1, ptr ad
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP_EXIT]] ], [ [[BUF1]], [[ENTRY]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]]
@@ -363,7 +363,7 @@ define float @sum_duplicate_preds(ptr addrspace(8) %buf, i32 %len) {
; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ [[SUM]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[I_NEXT]], [[LOOP]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[PTR]], [[LOOP]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[PTR]] = add i32 [[PTR_PREV_OFF]], 4
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
@@ -410,7 +410,7 @@ define float @sum_integer_ops_duplicate_preds(ptr addrspace(8) %buf, i32 %len) {
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[I_NEXT]], [[LOOP]] ]
; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP]] ], [ [[BUF]], [[ENTRY]] ], [ [[PTR_RSRC]], [[LOOP]] ]
; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[PTR_OFF]], [[LOOP]] ]
-; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]]
; CHECK-NEXT: [[PTR_PREV_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_PREV_RSRC]] to i160
; CHECK-NEXT: [[TMP0:%.*]] = shl nuw i160 [[PTR_PREV_INT_RSRC]], 32
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
index 167036eb7d725..6bd57ed0c07c2 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
@@ -23,72 +23,72 @@ define void @memcpy_known(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg %d
; CHECK: [[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0:![0-9]+]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
@@ -150,16 +150,16 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
@@ -169,16 +169,16 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
@@ -201,7 +201,7 @@ define void @memcpy_known_byte(ptr addrspace(7) inreg %src, ptr addrspace(7) inr
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
@@ -216,18 +216,18 @@ define void @memcpy_known_tail(ptr addrspace(7) inreg %src, ptr addrspace(7) inr
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 8
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 8
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw i32 [[SRC_OFF]], 12
-; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i32 [[DST_OFF]], 12
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i32 [[SRC_OFF]], 14
-; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[DST_OFF]], 14
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
@@ -248,196 +248,196 @@ define void @memcpy_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(7) inre
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[LOOP_INDEX_C:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX_C]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <64 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <64 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <64 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_3:%.*]] = insertelement <64 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = insertelement <64 x i32> [[DOTSLICE_3]], i32 [[DOTOFF_16]], i64 4
; CHECK-NEXT: [[DOTOFF_PTR_20:%.*]] = add i32 [[TMP1]], 20
-; CHECK-NEXT: [[DOTOFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_5:%.*]] = insertelement <64 x i32> [[DOTSLICE_4]], i32 [[DOTOFF_20]], i64 5
; CHECK-NEXT: [[DOTOFF_PTR_24:%.*]] = add i32 [[TMP1]], 24
-; CHECK-NEXT: [[DOTOFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_6:%.*]] = insertelement <64 x i32> [[DOTSLICE_5]], i32 [[DOTOFF_24]], i64 6
; CHECK-NEXT: [[DOTOFF_PTR_28:%.*]] = add i32 [[TMP1]], 28
-; CHECK-NEXT: [[DOTOFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_7:%.*]] = insertelement <64 x i32> [[DOTSLICE_6]], i32 [[DOTOFF_28]], i64 7
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = insertelement <64 x i32> [[DOTSLICE_7]], i32 [[DOTOFF_32]], i64 8
; CHECK-NEXT: [[DOTOFF_PTR_36:%.*]] = add i32 [[TMP1]], 36
-; CHECK-NEXT: [[DOTOFF_36:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_36:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_9:%.*]] = insertelement <64 x i32> [[DOTSLICE_8]], i32 [[DOTOFF_36]], i64 9
; CHECK-NEXT: [[DOTOFF_PTR_40:%.*]] = add i32 [[TMP1]], 40
-; CHECK-NEXT: [[DOTOFF_40:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_40:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_10:%.*]] = insertelement <64 x i32> [[DOTSLICE_9]], i32 [[DOTOFF_40]], i64 10
; CHECK-NEXT: [[DOTOFF_PTR_44:%.*]] = add i32 [[TMP1]], 44
-; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_11:%.*]] = insertelement <64 x i32> [[DOTSLICE_10]], i32 [[DOTOFF_44]], i64 11
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = insertelement <64 x i32> [[DOTSLICE_11]], i32 [[DOTOFF_48]], i64 12
; CHECK-NEXT: [[DOTOFF_PTR_52:%.*]] = add i32 [[TMP1]], 52
-; CHECK-NEXT: [[DOTOFF_52:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_52:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_13:%.*]] = insertelement <64 x i32> [[DOTSLICE_12]], i32 [[DOTOFF_52]], i64 13
; CHECK-NEXT: [[DOTOFF_PTR_56:%.*]] = add i32 [[TMP1]], 56
-; CHECK-NEXT: [[DOTOFF_56:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_56:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_14:%.*]] = insertelement <64 x i32> [[DOTSLICE_13]], i32 [[DOTOFF_56]], i64 14
; CHECK-NEXT: [[DOTOFF_PTR_60:%.*]] = add i32 [[TMP1]], 60
-; CHECK-NEXT: [[DOTOFF_60:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_60:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <64 x i32> [[DOTSLICE_14]], i32 [[DOTOFF_60]], i64 15
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = insertelement <64 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_64]], i64 16
; CHECK-NEXT: [[DOTOFF_PTR_68:%.*]] = add i32 [[TMP1]], 68
-; CHECK-NEXT: [[DOTOFF_68:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_68]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_68:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_68]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_17:%.*]] = insertelement <64 x i32> [[DOTSLICE_16]], i32 [[DOTOFF_68]], i64 17
; CHECK-NEXT: [[DOTOFF_PTR_72:%.*]] = add i32 [[TMP1]], 72
-; CHECK-NEXT: [[DOTOFF_72:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_72]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_72:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_72]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_18:%.*]] = insertelement <64 x i32> [[DOTSLICE_17]], i32 [[DOTOFF_72]], i64 18
; CHECK-NEXT: [[DOTOFF_PTR_76:%.*]] = add i32 [[TMP1]], 76
-; CHECK-NEXT: [[DOTOFF_76:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_76]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_76:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_76]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_19:%.*]] = insertelement <64 x i32> [[DOTSLICE_18]], i32 [[DOTOFF_76]], i64 19
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = insertelement <64 x i32> [[DOTSLICE_19]], i32 [[DOTOFF_80]], i64 20
; CHECK-NEXT: [[DOTOFF_PTR_84:%.*]] = add i32 [[TMP1]], 84
-; CHECK-NEXT: [[DOTOFF_84:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_84]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_84:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_84]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_21:%.*]] = insertelement <64 x i32> [[DOTSLICE_20]], i32 [[DOTOFF_84]], i64 21
; CHECK-NEXT: [[DOTOFF_PTR_88:%.*]] = add i32 [[TMP1]], 88
-; CHECK-NEXT: [[DOTOFF_88:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_88]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_88:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_88]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_22:%.*]] = insertelement <64 x i32> [[DOTSLICE_21]], i32 [[DOTOFF_88]], i64 22
; CHECK-NEXT: [[DOTOFF_PTR_92:%.*]] = add i32 [[TMP1]], 92
-; CHECK-NEXT: [[DOTOFF_92:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_92]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_92:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_92]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_23:%.*]] = insertelement <64 x i32> [[DOTSLICE_22]], i32 [[DOTOFF_92]], i64 23
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = insertelement <64 x i32> [[DOTSLICE_23]], i32 [[DOTOFF_96]], i64 24
; CHECK-NEXT: [[DOTOFF_PTR_100:%.*]] = add i32 [[TMP1]], 100
-; CHECK-NEXT: [[DOTOFF_100:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_100]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_100:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_100]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_25:%.*]] = insertelement <64 x i32> [[DOTSLICE_24]], i32 [[DOTOFF_100]], i64 25
; CHECK-NEXT: [[DOTOFF_PTR_104:%.*]] = add i32 [[TMP1]], 104
-; CHECK-NEXT: [[DOTOFF_104:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_104]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_104:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_104]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_26:%.*]] = insertelement <64 x i32> [[DOTSLICE_25]], i32 [[DOTOFF_104]], i64 26
; CHECK-NEXT: [[DOTOFF_PTR_108:%.*]] = add i32 [[TMP1]], 108
-; CHECK-NEXT: [[DOTOFF_108:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_108]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_108:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_108]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_27:%.*]] = insertelement <64 x i32> [[DOTSLICE_26]], i32 [[DOTOFF_108]], i64 27
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <64 x i32> [[DOTSLICE_27]], i32 [[DOTOFF_112]], i64 28
; CHECK-NEXT: [[DOTOFF_PTR_116:%.*]] = add i32 [[TMP1]], 116
-; CHECK-NEXT: [[DOTOFF_116:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_116]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_116:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_116]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_29:%.*]] = insertelement <64 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_116]], i64 29
; CHECK-NEXT: [[DOTOFF_PTR_120:%.*]] = add i32 [[TMP1]], 120
-; CHECK-NEXT: [[DOTOFF_120:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_120]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_120:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_120]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_30:%.*]] = insertelement <64 x i32> [[DOTSLICE_29]], i32 [[DOTOFF_120]], i64 30
; CHECK-NEXT: [[DOTOFF_PTR_124:%.*]] = add i32 [[TMP1]], 124
-; CHECK-NEXT: [[DOTOFF_124:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_124]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_124:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_124]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_31:%.*]] = insertelement <64 x i32> [[DOTSLICE_30]], i32 [[DOTOFF_124]], i64 31
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = insertelement <64 x i32> [[DOTSLICE_31]], i32 [[DOTOFF_128]], i64 32
; CHECK-NEXT: [[DOTOFF_PTR_132:%.*]] = add i32 [[TMP1]], 132
-; CHECK-NEXT: [[DOTOFF_132:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_132]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_132:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_132]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_33:%.*]] = insertelement <64 x i32> [[DOTSLICE_32]], i32 [[DOTOFF_132]], i64 33
; CHECK-NEXT: [[DOTOFF_PTR_136:%.*]] = add i32 [[TMP1]], 136
-; CHECK-NEXT: [[DOTOFF_136:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_136]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_136:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_136]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_34:%.*]] = insertelement <64 x i32> [[DOTSLICE_33]], i32 [[DOTOFF_136]], i64 34
; CHECK-NEXT: [[DOTOFF_PTR_140:%.*]] = add i32 [[TMP1]], 140
-; CHECK-NEXT: [[DOTOFF_140:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_140]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_140:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_140]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_35:%.*]] = insertelement <64 x i32> [[DOTSLICE_34]], i32 [[DOTOFF_140]], i64 35
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = insertelement <64 x i32> [[DOTSLICE_35]], i32 [[DOTOFF_144]], i64 36
; CHECK-NEXT: [[DOTOFF_PTR_148:%.*]] = add i32 [[TMP1]], 148
-; CHECK-NEXT: [[DOTOFF_148:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_148]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_148:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_148]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_37:%.*]] = insertelement <64 x i32> [[DOTSLICE_36]], i32 [[DOTOFF_148]], i64 37
; CHECK-NEXT: [[DOTOFF_PTR_152:%.*]] = add i32 [[TMP1]], 152
-; CHECK-NEXT: [[DOTOFF_152:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_152]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_152:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_152]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_38:%.*]] = insertelement <64 x i32> [[DOTSLICE_37]], i32 [[DOTOFF_152]], i64 38
; CHECK-NEXT: [[DOTOFF_PTR_156:%.*]] = add i32 [[TMP1]], 156
-; CHECK-NEXT: [[DOTOFF_156:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_156]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_156:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_156]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_39:%.*]] = insertelement <64 x i32> [[DOTSLICE_38]], i32 [[DOTOFF_156]], i64 39
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = insertelement <64 x i32> [[DOTSLICE_39]], i32 [[DOTOFF_160]], i64 40
; CHECK-NEXT: [[DOTOFF_PTR_164:%.*]] = add i32 [[TMP1]], 164
-; CHECK-NEXT: [[DOTOFF_164:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_164]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_164:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_164]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_41:%.*]] = insertelement <64 x i32> [[DOTSLICE_40]], i32 [[DOTOFF_164]], i64 41
; CHECK-NEXT: [[DOTOFF_PTR_168:%.*]] = add i32 [[TMP1]], 168
-; CHECK-NEXT: [[DOTOFF_168:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_168]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_168:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_168]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_42:%.*]] = insertelement <64 x i32> [[DOTSLICE_41]], i32 [[DOTOFF_168]], i64 42
; CHECK-NEXT: [[DOTOFF_PTR_172:%.*]] = add i32 [[TMP1]], 172
-; CHECK-NEXT: [[DOTOFF_172:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_172]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_172:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_172]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_43:%.*]] = insertelement <64 x i32> [[DOTSLICE_42]], i32 [[DOTOFF_172]], i64 43
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = insertelement <64 x i32> [[DOTSLICE_43]], i32 [[DOTOFF_176]], i64 44
; CHECK-NEXT: [[DOTOFF_PTR_180:%.*]] = add i32 [[TMP1]], 180
-; CHECK-NEXT: [[DOTOFF_180:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_180]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_180:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_180]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_45:%.*]] = insertelement <64 x i32> [[DOTSLICE_44]], i32 [[DOTOFF_180]], i64 45
; CHECK-NEXT: [[DOTOFF_PTR_184:%.*]] = add i32 [[TMP1]], 184
-; CHECK-NEXT: [[DOTOFF_184:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_184]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_184:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_184]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_46:%.*]] = insertelement <64 x i32> [[DOTSLICE_45]], i32 [[DOTOFF_184]], i64 46
; CHECK-NEXT: [[DOTOFF_PTR_188:%.*]] = add i32 [[TMP1]], 188
-; CHECK-NEXT: [[DOTOFF_188:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_188]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_188:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_188]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_47:%.*]] = insertelement <64 x i32> [[DOTSLICE_46]], i32 [[DOTOFF_188]], i64 47
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = insertelement <64 x i32> [[DOTSLICE_47]], i32 [[DOTOFF_192]], i64 48
; CHECK-NEXT: [[DOTOFF_PTR_196:%.*]] = add i32 [[TMP1]], 196
-; CHECK-NEXT: [[DOTOFF_196:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_196]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_196:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_196]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_49:%.*]] = insertelement <64 x i32> [[DOTSLICE_48]], i32 [[DOTOFF_196]], i64 49
; CHECK-NEXT: [[DOTOFF_PTR_200:%.*]] = add i32 [[TMP1]], 200
-; CHECK-NEXT: [[DOTOFF_200:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_200]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_200:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_200]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_50:%.*]] = insertelement <64 x i32> [[DOTSLICE_49]], i32 [[DOTOFF_200]], i64 50
; CHECK-NEXT: [[DOTOFF_PTR_204:%.*]] = add i32 [[TMP1]], 204
-; CHECK-NEXT: [[DOTOFF_204:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_204]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_204:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_204]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_51:%.*]] = insertelement <64 x i32> [[DOTSLICE_50]], i32 [[DOTOFF_204]], i64 51
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = insertelement <64 x i32> [[DOTSLICE_51]], i32 [[DOTOFF_208]], i64 52
; CHECK-NEXT: [[DOTOFF_PTR_212:%.*]] = add i32 [[TMP1]], 212
-; CHECK-NEXT: [[DOTOFF_212:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_212]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_212:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_212]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_53:%.*]] = insertelement <64 x i32> [[DOTSLICE_52]], i32 [[DOTOFF_212]], i64 53
; CHECK-NEXT: [[DOTOFF_PTR_216:%.*]] = add i32 [[TMP1]], 216
-; CHECK-NEXT: [[DOTOFF_216:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_216]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_216:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_216]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_54:%.*]] = insertelement <64 x i32> [[DOTSLICE_53]], i32 [[DOTOFF_216]], i64 54
; CHECK-NEXT: [[DOTOFF_PTR_220:%.*]] = add i32 [[TMP1]], 220
-; CHECK-NEXT: [[DOTOFF_220:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_220]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_220:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_220]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_55:%.*]] = insertelement <64 x i32> [[DOTSLICE_54]], i32 [[DOTOFF_220]], i64 55
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = insertelement <64 x i32> [[DOTSLICE_55]], i32 [[DOTOFF_224]], i64 56
; CHECK-NEXT: [[DOTOFF_PTR_228:%.*]] = add i32 [[TMP1]], 228
-; CHECK-NEXT: [[DOTOFF_228:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_228]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_228:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_228]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_57:%.*]] = insertelement <64 x i32> [[DOTSLICE_56]], i32 [[DOTOFF_228]], i64 57
; CHECK-NEXT: [[DOTOFF_PTR_232:%.*]] = add i32 [[TMP1]], 232
-; CHECK-NEXT: [[DOTOFF_232:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_232]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_232:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_232]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_58:%.*]] = insertelement <64 x i32> [[DOTSLICE_57]], i32 [[DOTOFF_232]], i64 58
; CHECK-NEXT: [[DOTOFF_PTR_236:%.*]] = add i32 [[TMP1]], 236
-; CHECK-NEXT: [[DOTOFF_236:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_236]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_236:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_236]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_59:%.*]] = insertelement <64 x i32> [[DOTSLICE_58]], i32 [[DOTOFF_236]], i64 59
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = insertelement <64 x i32> [[DOTSLICE_59]], i32 [[DOTOFF_240]], i64 60
; CHECK-NEXT: [[DOTOFF_PTR_244:%.*]] = add i32 [[TMP1]], 244
-; CHECK-NEXT: [[DOTOFF_244:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_244]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_244:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_244]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_61:%.*]] = insertelement <64 x i32> [[DOTSLICE_60]], i32 [[DOTOFF_244]], i64 61
; CHECK-NEXT: [[DOTOFF_PTR_248:%.*]] = add i32 [[TMP1]], 248
-; CHECK-NEXT: [[DOTOFF_248:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_248]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_248:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_248]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_62:%.*]] = insertelement <64 x i32> [[DOTSLICE_61]], i32 [[DOTOFF_248]], i64 62
; CHECK-NEXT: [[DOTOFF_PTR_252:%.*]] = add i32 [[TMP1]], 252
-; CHECK-NEXT: [[DOTOFF_252:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_252]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_252:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_252]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
@@ -585,16 +585,16 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
@@ -604,16 +604,16 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648, metadata [[META0]])
@@ -643,16 +643,16 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK: [[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0, metadata [[META0]])
@@ -672,9 +672,9 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]] ], [ [[TMP13:%.*]], %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP2]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[SRC_OFF]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP16:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[DST_OFF]], [[TMP9]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP13]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_LOOP_EXPANSION]]
@@ -762,67 +762,67 @@ define void @memcpy_known_p7_to_p1(ptr addrspace(7) inreg %src, ptr addrspace(1)
; CHECK: [[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i32 [[LOOP_INDEX]]
@@ -843,7 +843,7 @@ define void @memcpy_known_p7_to_p3(ptr addrspace(7) inreg %src, ptr addrspace(3)
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[SRC:%.*]], ptr addrspace(3) inreg [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 0
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr addrspace(3) [[TMP2]], align 16
; CHECK-NEXT: ret void
@@ -857,7 +857,7 @@ define void @memcpy_known_p7_to_p3_byte(ptr addrspace(7) inreg %src, ptr addrspa
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[SRC:%.*]], ptr addrspace(3) inreg [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 0
; CHECK-NEXT: store i32 [[TMP1]], ptr addrspace(3) [[TMP2]], align 16
; CHECK-NEXT: ret void
@@ -875,67 +875,67 @@ define void @memcpy_known_p7_to_p3_long(ptr addrspace(7) inreg %src, ptr addrspa
; CHECK: [[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[LOOP_INDEX]]
@@ -969,67 +969,67 @@ define void @memcpy.inline_known(ptr addrspace(7) inreg %src, ptr addrspace(7) i
; CHECK: [[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
@@ -1096,16 +1096,16 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
@@ -1115,16 +1115,16 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
@@ -1147,7 +1147,7 @@ define void @memcpy.inline_known_byte(ptr addrspace(7) inreg %src, ptr addrspace
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
@@ -1162,18 +1162,18 @@ define void @memcpy.inline_known_tail(ptr addrspace(7) inreg %src, ptr addrspace
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 8
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 8
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw i32 [[SRC_OFF]], 12
-; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i32 [[DST_OFF]], 12
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i32 [[SRC_OFF]], 14
-; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[DST_OFF]], 14
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
@@ -1194,196 +1194,196 @@ define void @memcpy.inline_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[LOOP_INDEX_C:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX_C]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <64 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <64 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <64 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_3:%.*]] = insertelement <64 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = insertelement <64 x i32> [[DOTSLICE_3]], i32 [[DOTOFF_16]], i64 4
; CHECK-NEXT: [[DOTOFF_PTR_20:%.*]] = add i32 [[TMP1]], 20
-; CHECK-NEXT: [[DOTOFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_5:%.*]] = insertelement <64 x i32> [[DOTSLICE_4]], i32 [[DOTOFF_20]], i64 5
; CHECK-NEXT: [[DOTOFF_PTR_24:%.*]] = add i32 [[TMP1]], 24
-; CHECK-NEXT: [[DOTOFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_6:%.*]] = insertelement <64 x i32> [[DOTSLICE_5]], i32 [[DOTOFF_24]], i64 6
; CHECK-NEXT: [[DOTOFF_PTR_28:%.*]] = add i32 [[TMP1]], 28
-; CHECK-NEXT: [[DOTOFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_7:%.*]] = insertelement <64 x i32> [[DOTSLICE_6]], i32 [[DOTOFF_28]], i64 7
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = insertelement <64 x i32> [[DOTSLICE_7]], i32 [[DOTOFF_32]], i64 8
; CHECK-NEXT: [[DOTOFF_PTR_36:%.*]] = add i32 [[TMP1]], 36
-; CHECK-NEXT: [[DOTOFF_36:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_36:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_9:%.*]] = insertelement <64 x i32> [[DOTSLICE_8]], i32 [[DOTOFF_36]], i64 9
; CHECK-NEXT: [[DOTOFF_PTR_40:%.*]] = add i32 [[TMP1]], 40
-; CHECK-NEXT: [[DOTOFF_40:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_40:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_10:%.*]] = insertelement <64 x i32> [[DOTSLICE_9]], i32 [[DOTOFF_40]], i64 10
; CHECK-NEXT: [[DOTOFF_PTR_44:%.*]] = add i32 [[TMP1]], 44
-; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_11:%.*]] = insertelement <64 x i32> [[DOTSLICE_10]], i32 [[DOTOFF_44]], i64 11
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = insertelement <64 x i32> [[DOTSLICE_11]], i32 [[DOTOFF_48]], i64 12
; CHECK-NEXT: [[DOTOFF_PTR_52:%.*]] = add i32 [[TMP1]], 52
-; CHECK-NEXT: [[DOTOFF_52:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_52:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_13:%.*]] = insertelement <64 x i32> [[DOTSLICE_12]], i32 [[DOTOFF_52]], i64 13
; CHECK-NEXT: [[DOTOFF_PTR_56:%.*]] = add i32 [[TMP1]], 56
-; CHECK-NEXT: [[DOTOFF_56:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_56:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_14:%.*]] = insertelement <64 x i32> [[DOTSLICE_13]], i32 [[DOTOFF_56]], i64 14
; CHECK-NEXT: [[DOTOFF_PTR_60:%.*]] = add i32 [[TMP1]], 60
-; CHECK-NEXT: [[DOTOFF_60:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_60:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <64 x i32> [[DOTSLICE_14]], i32 [[DOTOFF_60]], i64 15
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = insertelement <64 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_64]], i64 16
; CHECK-NEXT: [[DOTOFF_PTR_68:%.*]] = add i32 [[TMP1]], 68
-; CHECK-NEXT: [[DOTOFF_68:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_68]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_68:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_68]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_17:%.*]] = insertelement <64 x i32> [[DOTSLICE_16]], i32 [[DOTOFF_68]], i64 17
; CHECK-NEXT: [[DOTOFF_PTR_72:%.*]] = add i32 [[TMP1]], 72
-; CHECK-NEXT: [[DOTOFF_72:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_72]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_72:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_72]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_18:%.*]] = insertelement <64 x i32> [[DOTSLICE_17]], i32 [[DOTOFF_72]], i64 18
; CHECK-NEXT: [[DOTOFF_PTR_76:%.*]] = add i32 [[TMP1]], 76
-; CHECK-NEXT: [[DOTOFF_76:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_76]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_76:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_76]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_19:%.*]] = insertelement <64 x i32> [[DOTSLICE_18]], i32 [[DOTOFF_76]], i64 19
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = insertelement <64 x i32> [[DOTSLICE_19]], i32 [[DOTOFF_80]], i64 20
; CHECK-NEXT: [[DOTOFF_PTR_84:%.*]] = add i32 [[TMP1]], 84
-; CHECK-NEXT: [[DOTOFF_84:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_84]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_84:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_84]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_21:%.*]] = insertelement <64 x i32> [[DOTSLICE_20]], i32 [[DOTOFF_84]], i64 21
; CHECK-NEXT: [[DOTOFF_PTR_88:%.*]] = add i32 [[TMP1]], 88
-; CHECK-NEXT: [[DOTOFF_88:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_88]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_88:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_88]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_22:%.*]] = insertelement <64 x i32> [[DOTSLICE_21]], i32 [[DOTOFF_88]], i64 22
; CHECK-NEXT: [[DOTOFF_PTR_92:%.*]] = add i32 [[TMP1]], 92
-; CHECK-NEXT: [[DOTOFF_92:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_92]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_92:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_92]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_23:%.*]] = insertelement <64 x i32> [[DOTSLICE_22]], i32 [[DOTOFF_92]], i64 23
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = insertelement <64 x i32> [[DOTSLICE_23]], i32 [[DOTOFF_96]], i64 24
; CHECK-NEXT: [[DOTOFF_PTR_100:%.*]] = add i32 [[TMP1]], 100
-; CHECK-NEXT: [[DOTOFF_100:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_100]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_100:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_100]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_25:%.*]] = insertelement <64 x i32> [[DOTSLICE_24]], i32 [[DOTOFF_100]], i64 25
; CHECK-NEXT: [[DOTOFF_PTR_104:%.*]] = add i32 [[TMP1]], 104
-; CHECK-NEXT: [[DOTOFF_104:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_104]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_104:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_104]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_26:%.*]] = insertelement <64 x i32> [[DOTSLICE_25]], i32 [[DOTOFF_104]], i64 26
; CHECK-NEXT: [[DOTOFF_PTR_108:%.*]] = add i32 [[TMP1]], 108
-; CHECK-NEXT: [[DOTOFF_108:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_108]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_108:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_108]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_27:%.*]] = insertelement <64 x i32> [[DOTSLICE_26]], i32 [[DOTOFF_108]], i64 27
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <64 x i32> [[DOTSLICE_27]], i32 [[DOTOFF_112]], i64 28
; CHECK-NEXT: [[DOTOFF_PTR_116:%.*]] = add i32 [[TMP1]], 116
-; CHECK-NEXT: [[DOTOFF_116:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_116]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_116:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_116]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_29:%.*]] = insertelement <64 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_116]], i64 29
; CHECK-NEXT: [[DOTOFF_PTR_120:%.*]] = add i32 [[TMP1]], 120
-; CHECK-NEXT: [[DOTOFF_120:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_120]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_120:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_120]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_30:%.*]] = insertelement <64 x i32> [[DOTSLICE_29]], i32 [[DOTOFF_120]], i64 30
; CHECK-NEXT: [[DOTOFF_PTR_124:%.*]] = add i32 [[TMP1]], 124
-; CHECK-NEXT: [[DOTOFF_124:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_124]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_124:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_124]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_31:%.*]] = insertelement <64 x i32> [[DOTSLICE_30]], i32 [[DOTOFF_124]], i64 31
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = insertelement <64 x i32> [[DOTSLICE_31]], i32 [[DOTOFF_128]], i64 32
; CHECK-NEXT: [[DOTOFF_PTR_132:%.*]] = add i32 [[TMP1]], 132
-; CHECK-NEXT: [[DOTOFF_132:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_132]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_132:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_132]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_33:%.*]] = insertelement <64 x i32> [[DOTSLICE_32]], i32 [[DOTOFF_132]], i64 33
; CHECK-NEXT: [[DOTOFF_PTR_136:%.*]] = add i32 [[TMP1]], 136
-; CHECK-NEXT: [[DOTOFF_136:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_136]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_136:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_136]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_34:%.*]] = insertelement <64 x i32> [[DOTSLICE_33]], i32 [[DOTOFF_136]], i64 34
; CHECK-NEXT: [[DOTOFF_PTR_140:%.*]] = add i32 [[TMP1]], 140
-; CHECK-NEXT: [[DOTOFF_140:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_140]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_140:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_140]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_35:%.*]] = insertelement <64 x i32> [[DOTSLICE_34]], i32 [[DOTOFF_140]], i64 35
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = insertelement <64 x i32> [[DOTSLICE_35]], i32 [[DOTOFF_144]], i64 36
; CHECK-NEXT: [[DOTOFF_PTR_148:%.*]] = add i32 [[TMP1]], 148
-; CHECK-NEXT: [[DOTOFF_148:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_148]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_148:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_148]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_37:%.*]] = insertelement <64 x i32> [[DOTSLICE_36]], i32 [[DOTOFF_148]], i64 37
; CHECK-NEXT: [[DOTOFF_PTR_152:%.*]] = add i32 [[TMP1]], 152
-; CHECK-NEXT: [[DOTOFF_152:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_152]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_152:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_152]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_38:%.*]] = insertelement <64 x i32> [[DOTSLICE_37]], i32 [[DOTOFF_152]], i64 38
; CHECK-NEXT: [[DOTOFF_PTR_156:%.*]] = add i32 [[TMP1]], 156
-; CHECK-NEXT: [[DOTOFF_156:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_156]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_156:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_156]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_39:%.*]] = insertelement <64 x i32> [[DOTSLICE_38]], i32 [[DOTOFF_156]], i64 39
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = insertelement <64 x i32> [[DOTSLICE_39]], i32 [[DOTOFF_160]], i64 40
; CHECK-NEXT: [[DOTOFF_PTR_164:%.*]] = add i32 [[TMP1]], 164
-; CHECK-NEXT: [[DOTOFF_164:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_164]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_164:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_164]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_41:%.*]] = insertelement <64 x i32> [[DOTSLICE_40]], i32 [[DOTOFF_164]], i64 41
; CHECK-NEXT: [[DOTOFF_PTR_168:%.*]] = add i32 [[TMP1]], 168
-; CHECK-NEXT: [[DOTOFF_168:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_168]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_168:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_168]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_42:%.*]] = insertelement <64 x i32> [[DOTSLICE_41]], i32 [[DOTOFF_168]], i64 42
; CHECK-NEXT: [[DOTOFF_PTR_172:%.*]] = add i32 [[TMP1]], 172
-; CHECK-NEXT: [[DOTOFF_172:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_172]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_172:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_172]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_43:%.*]] = insertelement <64 x i32> [[DOTSLICE_42]], i32 [[DOTOFF_172]], i64 43
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = insertelement <64 x i32> [[DOTSLICE_43]], i32 [[DOTOFF_176]], i64 44
; CHECK-NEXT: [[DOTOFF_PTR_180:%.*]] = add i32 [[TMP1]], 180
-; CHECK-NEXT: [[DOTOFF_180:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_180]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_180:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_180]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_45:%.*]] = insertelement <64 x i32> [[DOTSLICE_44]], i32 [[DOTOFF_180]], i64 45
; CHECK-NEXT: [[DOTOFF_PTR_184:%.*]] = add i32 [[TMP1]], 184
-; CHECK-NEXT: [[DOTOFF_184:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_184]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_184:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_184]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_46:%.*]] = insertelement <64 x i32> [[DOTSLICE_45]], i32 [[DOTOFF_184]], i64 46
; CHECK-NEXT: [[DOTOFF_PTR_188:%.*]] = add i32 [[TMP1]], 188
-; CHECK-NEXT: [[DOTOFF_188:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_188]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_188:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_188]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_47:%.*]] = insertelement <64 x i32> [[DOTSLICE_46]], i32 [[DOTOFF_188]], i64 47
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = insertelement <64 x i32> [[DOTSLICE_47]], i32 [[DOTOFF_192]], i64 48
; CHECK-NEXT: [[DOTOFF_PTR_196:%.*]] = add i32 [[TMP1]], 196
-; CHECK-NEXT: [[DOTOFF_196:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_196]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_196:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_196]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_49:%.*]] = insertelement <64 x i32> [[DOTSLICE_48]], i32 [[DOTOFF_196]], i64 49
; CHECK-NEXT: [[DOTOFF_PTR_200:%.*]] = add i32 [[TMP1]], 200
-; CHECK-NEXT: [[DOTOFF_200:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_200]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_200:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_200]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_50:%.*]] = insertelement <64 x i32> [[DOTSLICE_49]], i32 [[DOTOFF_200]], i64 50
; CHECK-NEXT: [[DOTOFF_PTR_204:%.*]] = add i32 [[TMP1]], 204
-; CHECK-NEXT: [[DOTOFF_204:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_204]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_204:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_204]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_51:%.*]] = insertelement <64 x i32> [[DOTSLICE_50]], i32 [[DOTOFF_204]], i64 51
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = insertelement <64 x i32> [[DOTSLICE_51]], i32 [[DOTOFF_208]], i64 52
; CHECK-NEXT: [[DOTOFF_PTR_212:%.*]] = add i32 [[TMP1]], 212
-; CHECK-NEXT: [[DOTOFF_212:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_212]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_212:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_212]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_53:%.*]] = insertelement <64 x i32> [[DOTSLICE_52]], i32 [[DOTOFF_212]], i64 53
; CHECK-NEXT: [[DOTOFF_PTR_216:%.*]] = add i32 [[TMP1]], 216
-; CHECK-NEXT: [[DOTOFF_216:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_216]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_216:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_216]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_54:%.*]] = insertelement <64 x i32> [[DOTSLICE_53]], i32 [[DOTOFF_216]], i64 54
; CHECK-NEXT: [[DOTOFF_PTR_220:%.*]] = add i32 [[TMP1]], 220
-; CHECK-NEXT: [[DOTOFF_220:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_220]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_220:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_220]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_55:%.*]] = insertelement <64 x i32> [[DOTSLICE_54]], i32 [[DOTOFF_220]], i64 55
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = insertelement <64 x i32> [[DOTSLICE_55]], i32 [[DOTOFF_224]], i64 56
; CHECK-NEXT: [[DOTOFF_PTR_228:%.*]] = add i32 [[TMP1]], 228
-; CHECK-NEXT: [[DOTOFF_228:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_228]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_228:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_228]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_57:%.*]] = insertelement <64 x i32> [[DOTSLICE_56]], i32 [[DOTOFF_228]], i64 57
; CHECK-NEXT: [[DOTOFF_PTR_232:%.*]] = add i32 [[TMP1]], 232
-; CHECK-NEXT: [[DOTOFF_232:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_232]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_232:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_232]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_58:%.*]] = insertelement <64 x i32> [[DOTSLICE_57]], i32 [[DOTOFF_232]], i64 58
; CHECK-NEXT: [[DOTOFF_PTR_236:%.*]] = add i32 [[TMP1]], 236
-; CHECK-NEXT: [[DOTOFF_236:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_236]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_236:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_236]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_59:%.*]] = insertelement <64 x i32> [[DOTSLICE_58]], i32 [[DOTOFF_236]], i64 59
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = insertelement <64 x i32> [[DOTSLICE_59]], i32 [[DOTOFF_240]], i64 60
; CHECK-NEXT: [[DOTOFF_PTR_244:%.*]] = add i32 [[TMP1]], 244
-; CHECK-NEXT: [[DOTOFF_244:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_244]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_244:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_244]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_61:%.*]] = insertelement <64 x i32> [[DOTSLICE_60]], i32 [[DOTOFF_244]], i64 61
; CHECK-NEXT: [[DOTOFF_PTR_248:%.*]] = add i32 [[TMP1]], 248
-; CHECK-NEXT: [[DOTOFF_248:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_248]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_248:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_248]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_62:%.*]] = insertelement <64 x i32> [[DOTSLICE_61]], i32 [[DOTOFF_248]], i64 62
; CHECK-NEXT: [[DOTOFF_PTR_252:%.*]] = add i32 [[TMP1]], 252
-; CHECK-NEXT: [[DOTOFF_252:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_252]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_252:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_252]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
@@ -1531,16 +1531,16 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
@@ -1550,16 +1550,16 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648, metadata [[META0]])
@@ -1589,16 +1589,16 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK: [[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0, metadata [[META0]])
@@ -1618,9 +1618,9 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]] ], [ [[TMP13:%.*]], %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP2]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[SRC_OFF]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP16:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[DST_OFF]], [[TMP9]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP13]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_LOOP_EXPANSION]]
@@ -1708,67 +1708,67 @@ define void @memcpy.inline_known_p7_to_p1(ptr addrspace(7) inreg %src, ptr addrs
; CHECK: [[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i32 [[LOOP_INDEX]]
@@ -1789,7 +1789,7 @@ define void @memcpy.inline_known_p7_to_p3(ptr addrspace(7) inreg %src, ptr addrs
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[SRC:%.*]], ptr addrspace(3) inreg [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 0
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr addrspace(3) [[TMP2]], align 16
; CHECK-NEXT: ret void
@@ -1803,7 +1803,7 @@ define void @memcpy.inline_known_p7_to_p3_byte(ptr addrspace(7) inreg %src, ptr
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[SRC:%.*]], ptr addrspace(3) inreg [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 0
; CHECK-NEXT: store i32 [[TMP1]], ptr addrspace(3) [[TMP2]], align 16
; CHECK-NEXT: ret void
@@ -1821,67 +1821,67 @@ define void @memcpy.inline_known_p7_to_p3_long(ptr addrspace(7) inreg %src, ptr
; CHECK: [[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[LOOP_INDEX]]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
index de955eebd1518..d8d40bf1b2015 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
@@ -7,19 +7,19 @@ target triple = "amdgpu9.00--"
define void @loads(ptr addrspace(8) %buf) {
; CHECK-LABEL: define void @loads
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]]) {
-; CHECK-NEXT: [[SCALAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: [[VEC2:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: [[VEC4:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: [[NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !nontemporal [[META0:![0-9]+]]
-; CHECK-NEXT: [[INVARIANT:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !invariant.load [[META1:![0-9]+]]
-; CHECK-NEXT: [[NONTEMPORAL_INVARIANT:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !invariant.load [[META1]], !nontemporal [[META0]]
-; CHECK-NEXT: [[VOLATILE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
-; CHECK-NEXT: [[VOLATILE_NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648), !nontemporal [[META0]]
+; CHECK-NEXT: [[SCALAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0:![0-9]+]])
+; CHECK-NEXT: [[VEC2:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: [[VEC4:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: [[NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]]), !nontemporal [[META1:![0-9]+]]
+; CHECK-NEXT: [[INVARIANT:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]]), !invariant.load [[META0]]
+; CHECK-NEXT: [[NONTEMPORAL_INVARIANT:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]]), !invariant.load [[META0]], !nontemporal [[META1]]
+; CHECK-NEXT: [[VOLATILE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META0]])
+; CHECK-NEXT: [[VOLATILE_NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META0]]), !nontemporal [[META1]]
; CHECK-NEXT: fence syncscope("wavefront") release
; CHECK-NEXT: [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2:![0-9]+]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
-; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
-; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3:![0-9]+]])
+; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3:![0-9]+]])
+; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META4:![0-9]+]])
; CHECK-NEXT: fence acquire
; CHECK-NEXT: ret void
;
@@ -47,17 +47,17 @@ define void @loads(ptr addrspace(8) %buf) {
define void @stores(ptr addrspace(8) %buf, float %f, <4 x float> %f4) {
; CHECK-LABEL: define void @stores
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], <4 x float> [[F4:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]]), !nontemporal [[META0]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META1]])
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META1]]), !nontemporal [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META0]]), !nontemporal [[META1]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META0]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META0]]), !nontemporal [[META1]]
; CHECK-NEXT: fence syncscope("wavefront") release
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
-; CHECK-NEXT: fence release
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3]])
+; CHECK-NEXT: fence release
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META5:![0-9]+]])
; CHECK-NEXT: ret void
;
%base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -154,7 +154,7 @@ define {i32, i1} @cmpxchg(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
; CHECK-LABEL: define { i32, i1 } @cmpxchg
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2]])
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META6:![0-9]+]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]]
@@ -172,7 +172,7 @@ define {i32, i1} @cmpxchg_weak(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
; CHECK-LABEL: define { i32, i1 } @cmpxchg_weak
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META6]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll
index dca94fd7db941..0103de8ba5382 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll
@@ -615,7 +615,7 @@ define i32 @invariant_start_end(ptr addrspace(7) %p) {
; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0
; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1
; CHECK-NEXT: [[INV:%.*]] = call ptr @llvm.invariant.start.p8(i64 256, ptr addrspace(8) [[P_RSRC]])
-; CHECK-NEXT: [[V:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[V:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: call void @llvm.invariant.end.p8(ptr [[INV]], i64 256, ptr addrspace(8) [[P_RSRC]])
; CHECK-NEXT: ret i32 [[V]]
;
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
index 76fe2f662b393..9ce2550c0ccb5 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -45,7 +45,7 @@ define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace
; CHECK-NEXT: [[BUF_PTR_4_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_4]] to i32, !dbg [[DBG32]]
; CHECK-NEXT: #dbg_value(i32 [[BUF_PTR_4_PTR_OFF]], [[META19:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG32]])
; CHECK-NEXT: #dbg_value(ptr addrspace(8) [[BUF_PTR_4_PTR_RSRC]], [[META19]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG32]])
-; CHECK-NEXT: [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF_PTR_4_PTR_RSRC]], i32 [[BUF_PTR_4_PTR_OFF]], i32 0, i32 0), !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT: [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF_PTR_4_PTR_RSRC]], i32 [[BUF_PTR_4_PTR_OFF]], i32 0, i32 0, metadata [[META8:![0-9]+]]), !dbg [[DBG33:![0-9]+]]
; CHECK-NEXT: #dbg_value(float [[RET]], [[META20:![0-9]+]], !DIExpression(), [[DBG33]])
; CHECK-NEXT: [[AUX_PTR_2:%.*]] = load i160, ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG34:![0-9]+]]
; CHECK-NEXT: [[TMP6:%.*]] = lshr i160 [[AUX_PTR_2]], 32, !dbg [[DBG34]]
@@ -56,7 +56,7 @@ define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace
; CHECK-NEXT: #dbg_value(ptr addrspace(8) [[AUX_PTR_2_PTR_RSRC]], [[META21]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG34]])
; CHECK-NEXT: [[BUF_PTR_4_LEGAL:%.*]] = bitcast i160 [[BUF_PTR_4]] to <5 x i32>, !dbg [[DBG35:![0-9]+]]
; CHECK-NEXT: [[BUF_PTR_4_SLICE_0:%.*]] = shufflevector <5 x i32> [[BUF_PTR_4_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, !dbg [[DBG35]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0, metadata [[META8:![0-9]+]]), !dbg [[DBG35]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0, metadata [[META8]]), !dbg [[DBG35]]
; CHECK-NEXT: [[AUX_PTR_2_PTR_PART_4:%.*]] = add nuw i32 [[AUX_PTR_2_PTR_OFF]], 16, !dbg [[DBG35]]
; CHECK-NEXT: [[BUF_PTR_4_SLICE_4:%.*]] = extractelement <5 x i32> [[BUF_PTR_4_LEGAL]], i64 4, !dbg [[DBG35]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0, metadata [[META8]]), !dbg [[DBG35]]
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
index 6523c168dd50c..169e13c87d63e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
@@ -81,9 +81,16 @@ define void @buffer_fat_ptr_seq_cst_atomic_add_noret_i32(ptr addrspace(7) inreg
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -123,8 +130,15 @@ define i32 @buffer_fat_ptr_seq_cst_cmpxchg_i32(ptr addrspace(7) inreg %ptr, i32
; GFX1250-NEXT: s_wait_xcnt 0x1
; GFX1250-NEXT: v_mov_b32_e32 v1, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, s16
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -161,7 +175,11 @@ define i32 @buffer_fat_ptr_seq_cst_load_i32(ptr addrspace(7) inreg %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v0, s16
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
@@ -196,7 +214,12 @@ define void @buffer_fat_ptr_seq_cst_store_i32(ptr addrspace(7) inreg %ptr, i32 %
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
diff --git a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
index f71844626cc91..9e4bdb6276587 100644
--- a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
+++ b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
@@ -8,8 +8,8 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
; FNATTRS-SAME: (ptr nofree readonly captures(none) [[P:%.*]], ptr nofree writeonly captures(none) [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
; FNATTRS-NEXT: [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[P]], i16 0, i64 4, i32 822243328)
; FNATTRS-NEXT: [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[Q]], i16 0, i64 4, i32 822243328)
-; FNATTRS-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[P_RSRC]], i32 0, i32 0, i32 0)
-; FNATTRS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
+; FNATTRS-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[P_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
+; FNATTRS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0]])
; FNATTRS-NEXT: ret void
;
; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
@@ -17,8 +17,8 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
; ATTRIBUTOR-SAME: (ptr nofree readonly captures(none) [[P:%.*]], ptr nofree writeonly captures(none) [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
; ATTRIBUTOR-NEXT: [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[P]], i16 0, i64 4, i32 822243328) #[[ATTR4:[0-9]+]]
; ATTRIBUTOR-NEXT: [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[Q]], i16 0, i64 4, i32 822243328) #[[ATTR4]]
-; ATTRIBUTOR-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) readonly captures(none) [[P_RSRC]], i32 0, i32 0, i32 0) #[[ATTR5:[0-9]+]]
-; ATTRIBUTOR-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]]) #[[ATTR6:[0-9]+]]
+; ATTRIBUTOR-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) readonly captures(none) [[P_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]]) #[[ATTR5:[0-9]+]]
+; ATTRIBUTOR-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0]]) #[[ATTR6:[0-9]+]]
; ATTRIBUTOR-NEXT: ret void
;
%p.rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 0, i64 4, i32 822243328)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
index 73d8f03784014..a99657f633d20 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
@@ -408,7 +408,7 @@ declare <4 x i8> @llvm.amdgcn.raw.buffer.load.v4i8(<4 x i32>, i32, i32, i32) #1
define amdgpu_ps float @raw_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1:![0-9]+]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -417,7 +417,7 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i3
define amdgpu_ps <1 x float> @raw_ptr_buffer_load_v1f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.v1f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -426,7 +426,7 @@ define amdgpu_ps <1 x float> @raw_ptr_buffer_load_v1f32(ptr addrspace(8) inreg %
define amdgpu_ps <2 x float> @raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -435,7 +435,7 @@ define amdgpu_ps <2 x float> @raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %
define amdgpu_ps <4 x float> @raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -444,7 +444,7 @@ define amdgpu_ps <4 x float> @raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -455,7 +455,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v2f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v2f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -465,7 +465,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v2f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -476,7 +476,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -487,7 +487,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -498,7 +498,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 12
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -508,7 +508,7 @@ define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -519,7 +519,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v4f32(ptr ad
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -530,7 +530,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v4f32(ptr ad
define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -540,7 +540,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr ad
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -551,7 +551,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_v4f32(p
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -561,7 +561,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_v4f32(p
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -572,7 +572,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_v4f32(p
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -583,7 +583,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v3f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -594,7 +594,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v3f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -604,7 +604,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v3f32(ptr addrspace(8)
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -615,7 +615,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v3f32(ptr ad
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -625,7 +625,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v3f32(ptr ad
define i32 @extract0_bitcast_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -637,7 +637,7 @@ define i32 @extract0_bitcast_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %r
define float @extract0_bitcast_raw_ptr_buffer_load_v4i32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_v4i32(
-; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast i32 [[VAR]] to float
; CHECK-NEXT: ret float [[VAR2]]
;
@@ -649,7 +649,7 @@ define float @extract0_bitcast_raw_ptr_buffer_load_v4i32(ptr addrspace(8) inreg
define amdgpu_ps float @preserve_metadata_extract_elt0_raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -667,7 +667,7 @@ declare <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8), i32,
define amdgpu_ps half @extract_elt0_raw_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v2f16(
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -678,7 +678,7 @@ define amdgpu_ps half @extract_elt0_raw_ptr_buffer_load_v2f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v2f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -689,7 +689,7 @@ define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v2f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v3f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v3f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <3 x half> @llvm.amdgcn.raw.ptr.buffer.load.v3f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -700,7 +700,7 @@ define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v3f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -711,7 +711,7 @@ define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v4f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt3_raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 6
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -721,7 +721,7 @@ define amdgpu_ps half @extract_elt3_raw_ptr_buffer_load_v4f16(ptr addrspace(8) i
define amdgpu_ps <2 x half> @extract_elt0_elt1_raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v4f16(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x half> [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -736,7 +736,7 @@ declare <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8), i32,
define amdgpu_ps i8 @extract_elt0_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v2i8(
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -747,7 +747,7 @@ define amdgpu_ps i8 @extract_elt0_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v2i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -758,7 +758,7 @@ define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v3i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v3i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <3 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v3i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -769,7 +769,7 @@ define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v3i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -780,7 +780,7 @@ define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt3_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 3
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -790,7 +790,7 @@ define amdgpu_ps i8 @extract_elt3_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inre
define amdgpu_ps <2 x i8> @extract_elt0_elt1_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v4i8(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i8> [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1202,7 +1202,7 @@ declare <4 x i8> @llvm.amdgcn.s.buffer.load.v4i8(<4 x i32>, i32, i32) #1
define amdgpu_ps float @ptr_s_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i32 %ofs) #0 {
; CHECK-LABEL: @ptr_s_buffer_load_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.ptr.s.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 0), !invariant.load [[META1:![0-9]+]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.ptr.s.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 0), !invariant.load [[META1]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.ptr.s.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 0), !invariant.load !1
@@ -1854,7 +1854,7 @@ declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i3
define amdgpu_ps float @raw_ptr_buffer_load_format_f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1863,7 +1863,7 @@ define amdgpu_ps float @raw_ptr_buffer_load_format_f32(ptr addrspace(8) inreg %r
define amdgpu_ps <1 x float> @raw_ptr_buffer_load_format_v1f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v1f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1872,7 +1872,7 @@ define amdgpu_ps <1 x float> @raw_ptr_buffer_load_format_v1f32(ptr addrspace(8)
define amdgpu_ps <2 x float> @raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1881,7 +1881,7 @@ define amdgpu_ps <2 x float> @raw_ptr_buffer_load_format_v2f32(ptr addrspace(8)
define amdgpu_ps <4 x float> @raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1890,7 +1890,7 @@ define amdgpu_ps <4 x float> @raw_ptr_buffer_load_format_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1900,7 +1900,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v2f32(ptr addrsp
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1911,7 +1911,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v2f32(ptr addrsp
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1921,7 +1921,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1932,7 +1932,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1943,7 +1943,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1954,7 +1954,7 @@ define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1964,7 +1964,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v4f32
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -1975,7 +1975,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v4f32
define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -1986,7 +1986,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_format_v4f32
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1996,7 +1996,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_format_
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -2007,7 +2007,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_format_
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -2018,7 +2018,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_format_
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -2028,7 +2028,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v3f32(ptr addrsp
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -2039,7 +2039,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v3f32(ptr addrsp
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -2050,7 +2050,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v3f32(ptr addrsp
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -2060,7 +2060,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v3f32
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -2071,7 +2071,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v3f32
define i32 @extract0_bitcast_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -2083,7 +2083,7 @@ define i32 @extract0_bitcast_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) i
define float @extract0_bitcast_raw_ptr_buffer_load_format_v4i32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_format_v4i32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[VAR]]
;
%var = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -2093,7 +2093,7 @@ define float @extract0_bitcast_raw_ptr_buffer_load_format_v4i32(ptr addrspace(8)
define amdgpu_ps float @preserve_metadata_extract_elt0_raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -2514,7 +2514,7 @@ declare <4 x i8> @llvm.amdgcn.struct.buffer.load.v4i8(<4 x i32>, i32, i32, i32,
define amdgpu_ps float @struct_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2523,7 +2523,7 @@ define amdgpu_ps float @struct_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc,
define amdgpu_ps <1 x float> @struct_ptr_buffer_load_v1f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.v1f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2532,7 +2532,7 @@ define amdgpu_ps <1 x float> @struct_ptr_buffer_load_v1f32(ptr addrspace(8) inre
define amdgpu_ps <2 x float> @struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2541,7 +2541,7 @@ define amdgpu_ps <2 x float> @struct_ptr_buffer_load_v2f32(ptr addrspace(8) inre
define amdgpu_ps <4 x float> @struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2550,7 +2550,7 @@ define amdgpu_ps <4 x float> @struct_ptr_buffer_load_v4f32(ptr addrspace(8) inre
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2561,7 +2561,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v2f32(ptr addrspace(
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v2f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2571,7 +2571,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v2f32(ptr addrspace(
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2582,7 +2582,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2593,7 +2593,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2604,7 +2604,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 12
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2614,7 +2614,7 @@ define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2625,7 +2625,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v4f32(ptr
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2636,7 +2636,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v4f32(ptr
define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2646,7 +2646,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2657,7 +2657,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_v4f3
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2667,7 +2667,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_v4f3
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -2678,7 +2678,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_v4f3
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2689,7 +2689,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v3f32(ptr addrspace(
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2700,7 +2700,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v3f32(ptr addrspace(
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2710,7 +2710,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v3f32(ptr addrspace(
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2721,7 +2721,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v3f32(ptr
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2731,7 +2731,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v3f32(ptr
define i32 @extract0_bitcast_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -2743,7 +2743,7 @@ define i32 @extract0_bitcast_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg
define float @extract0_bitcast_struct_ptr_buffer_load_v4i32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_struct_ptr_buffer_load_v4i32(
-; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast i32 [[VAR]] to float
; CHECK-NEXT: ret float [[VAR2]]
;
@@ -2755,7 +2755,7 @@ define float @extract0_bitcast_struct_ptr_buffer_load_v4i32(ptr addrspace(8) inr
define amdgpu_ps float @preserve_metadata_extract_elt0_struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -2773,7 +2773,7 @@ declare <4 x i32> @llvm.amdgcn.struct.ptr.buffer.load.v4i32(ptr addrspace(8), i3
define amdgpu_ps half @extract_elt0_struct_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v2f16(
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2784,7 +2784,7 @@ define amdgpu_ps half @extract_elt0_struct_ptr_buffer_load_v2f16(ptr addrspace(8
define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v2f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2795,7 +2795,7 @@ define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v2f16(ptr addrspace(8
define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v3f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v3f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.v3f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2806,7 +2806,7 @@ define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v3f16(ptr addrspace(8
define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2817,7 +2817,7 @@ define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v4f16(ptr addrspace(8
define amdgpu_ps half @extract_elt3_struct_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 6
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2827,7 +2827,7 @@ define amdgpu_ps half @extract_elt3_struct_ptr_buffer_load_v4f16(ptr addrspace(8
define amdgpu_ps <2 x half> @extract_elt0_elt1_struct_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v4f16(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x half> [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2842,7 +2842,7 @@ declare <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8), i
define amdgpu_ps i8 @extract_elt0_struct_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v2i8(
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2853,7 +2853,7 @@ define amdgpu_ps i8 @extract_elt0_struct_ptr_buffer_load_v2i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v2i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2864,7 +2864,7 @@ define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v2i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v3i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v3i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <3 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v3i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2875,7 +2875,7 @@ define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v3i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2886,7 +2886,7 @@ define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v4i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt3_struct_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 3
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2896,7 +2896,7 @@ define amdgpu_ps i8 @extract_elt3_struct_ptr_buffer_load_v4i8(ptr addrspace(8) i
define amdgpu_ps <2 x i8> @extract_elt0_elt1_struct_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v4i8(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i8> [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3166,7 +3166,7 @@ declare <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32>, i32, i
define amdgpu_ps float @struct_ptr_buffer_load_format_f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3175,7 +3175,7 @@ define amdgpu_ps float @struct_ptr_buffer_load_format_f32(ptr addrspace(8) inreg
define amdgpu_ps <1 x float> @struct_ptr_buffer_load_format_v1f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v1f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3184,7 +3184,7 @@ define amdgpu_ps <1 x float> @struct_ptr_buffer_load_format_v1f32(ptr addrspace(
define amdgpu_ps <2 x float> @struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3193,7 +3193,7 @@ define amdgpu_ps <2 x float> @struct_ptr_buffer_load_format_v2f32(ptr addrspace(
define amdgpu_ps <4 x float> @struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3202,7 +3202,7 @@ define amdgpu_ps <4 x float> @struct_ptr_buffer_load_format_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3212,7 +3212,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v2f32(ptr add
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3223,7 +3223,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v2f32(ptr add
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3233,7 +3233,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3244,7 +3244,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3255,7 +3255,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3266,7 +3266,7 @@ define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3276,7 +3276,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v4
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -3287,7 +3287,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v4
define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -3298,7 +3298,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_format_v4
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3308,7 +3308,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_form
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -3319,7 +3319,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_form
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -3330,7 +3330,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_form
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3340,7 +3340,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v3f32(ptr add
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3351,7 +3351,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v3f32(ptr add
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3362,7 +3362,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v3f32(ptr add
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3372,7 +3372,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v3
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -3383,7 +3383,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v3
define i32 @extract0_bitcast_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -3395,7 +3395,7 @@ define i32 @extract0_bitcast_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8
define amdgpu_ps float @preserve_metadata_extract_elt0_struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index f8cebe1c42aae..769f16ef69f93 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -408,7 +408,7 @@ declare <4 x i8> @llvm.amdgcn.raw.buffer.load.v4i8(<4 x i32>, i32, i32, i32) #1
define amdgpu_ps float @raw_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1:![0-9]+]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -417,7 +417,7 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i3
define amdgpu_ps <1 x float> @raw_ptr_buffer_load_v1f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.v1f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -426,7 +426,7 @@ define amdgpu_ps <1 x float> @raw_ptr_buffer_load_v1f32(ptr addrspace(8) inreg %
define amdgpu_ps <2 x float> @raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -435,7 +435,7 @@ define amdgpu_ps <2 x float> @raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %
define amdgpu_ps <4 x float> @raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -444,7 +444,7 @@ define amdgpu_ps <4 x float> @raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -455,7 +455,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v2f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v2f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -465,7 +465,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v2f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -476,7 +476,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -487,7 +487,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -498,7 +498,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 12
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -508,7 +508,7 @@ define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8)
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -519,7 +519,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v4f32(ptr ad
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -530,7 +530,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v4f32(ptr ad
define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -540,7 +540,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr ad
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -551,7 +551,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_v4f32(p
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -561,7 +561,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_v4f32(p
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -572,7 +572,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_v4f32(p
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -583,7 +583,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_v3f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -594,7 +594,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_v3f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -604,7 +604,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_v3f32(ptr addrspace(8)
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -615,7 +615,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_v3f32(ptr ad
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -625,7 +625,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_v3f32(ptr ad
define i32 @extract0_bitcast_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -637,7 +637,7 @@ define i32 @extract0_bitcast_raw_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %r
define float @extract0_bitcast_raw_ptr_buffer_load_v4i32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_v4i32(
-; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast i32 [[VAR]] to float
; CHECK-NEXT: ret float [[VAR2]]
;
@@ -649,7 +649,7 @@ define float @extract0_bitcast_raw_ptr_buffer_load_v4i32(ptr addrspace(8) inreg
define amdgpu_ps float @preserve_metadata_extract_elt0_raw_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -667,7 +667,7 @@ declare <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8), i32,
define amdgpu_ps half @extract_elt0_raw_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v2f16(
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -678,7 +678,7 @@ define amdgpu_ps half @extract_elt0_raw_ptr_buffer_load_v2f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v2f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -689,7 +689,7 @@ define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v2f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v3f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v3f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <3 x half> @llvm.amdgcn.raw.ptr.buffer.load.v3f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -700,7 +700,7 @@ define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v3f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -711,7 +711,7 @@ define amdgpu_ps half @extract_elt1_raw_ptr_buffer_load_v4f16(ptr addrspace(8) i
define amdgpu_ps half @extract_elt3_raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 6
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -721,7 +721,7 @@ define amdgpu_ps half @extract_elt3_raw_ptr_buffer_load_v4f16(ptr addrspace(8) i
define amdgpu_ps <2 x half> @extract_elt0_elt1_raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v4f16(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x half> [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -736,7 +736,7 @@ declare <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8), i32,
define amdgpu_ps i8 @extract_elt0_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_v2i8(
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -747,7 +747,7 @@ define amdgpu_ps i8 @extract_elt0_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v2i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -758,7 +758,7 @@ define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v2i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v3i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v3i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <3 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v3i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -769,7 +769,7 @@ define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v3i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -780,7 +780,7 @@ define amdgpu_ps i8 @extract_elt1_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inre
define amdgpu_ps i8 @extract_elt3_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 3
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -790,7 +790,7 @@ define amdgpu_ps i8 @extract_elt3_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inre
define amdgpu_ps <2 x i8> @extract_elt0_elt1_raw_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_v4i8(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i8> [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.raw.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1202,7 +1202,7 @@ declare <4 x i8> @llvm.amdgcn.s.buffer.load.v4i8(<4 x i32>, i32, i32) #1
define amdgpu_ps float @ptr_s_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i32 %ofs) #0 {
; CHECK-LABEL: @ptr_s_buffer_load_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.ptr.s.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 0), !invariant.load [[META1:![0-9]+]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.ptr.s.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 0), !invariant.load [[META1]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.ptr.s.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 0), !invariant.load !1
@@ -1854,7 +1854,7 @@ declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i3
define amdgpu_ps float @raw_ptr_buffer_load_format_f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1863,7 +1863,7 @@ define amdgpu_ps float @raw_ptr_buffer_load_format_f32(ptr addrspace(8) inreg %r
define amdgpu_ps <1 x float> @raw_ptr_buffer_load_format_v1f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v1f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1872,7 +1872,7 @@ define amdgpu_ps <1 x float> @raw_ptr_buffer_load_format_v1f32(ptr addrspace(8)
define amdgpu_ps <2 x float> @raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1881,7 +1881,7 @@ define amdgpu_ps <2 x float> @raw_ptr_buffer_load_format_v2f32(ptr addrspace(8)
define amdgpu_ps <4 x float> @raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1890,7 +1890,7 @@ define amdgpu_ps <4 x float> @raw_ptr_buffer_load_format_v4f32(ptr addrspace(8)
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1900,7 +1900,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v2f32(ptr addrsp
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1911,7 +1911,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v2f32(ptr addrsp
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1921,7 +1921,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1932,7 +1932,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1943,7 +1943,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -1954,7 +1954,7 @@ define amdgpu_ps float @extract_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrsp
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1964,7 +1964,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v4f32
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -1975,7 +1975,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v4f32
define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -1986,7 +1986,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_ptr_buffer_load_format_v4f32
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -1996,7 +1996,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_ptr_buffer_load_format_
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -2007,7 +2007,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_ptr_buffer_load_format_
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -2018,7 +2018,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_ptr_buffer_load_format_
define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -2028,7 +2028,7 @@ define amdgpu_ps float @extract_elt0_raw_ptr_buffer_load_format_v3f32(ptr addrsp
define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -2039,7 +2039,7 @@ define amdgpu_ps float @extract_elt1_raw_ptr_buffer_load_format_v3f32(ptr addrsp
define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -2050,7 +2050,7 @@ define amdgpu_ps float @extract_elt2_raw_ptr_buffer_load_format_v3f32(ptr addrsp
define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -2060,7 +2060,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_ptr_buffer_load_format_v3f32
define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_raw_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -2071,7 +2071,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_ptr_buffer_load_format_v3f32
define i32 @extract0_bitcast_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -2083,7 +2083,7 @@ define i32 @extract0_bitcast_raw_ptr_buffer_load_format_v4f32(ptr addrspace(8) i
define float @extract0_bitcast_raw_ptr_buffer_load_format_v4i32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_raw_ptr_buffer_load_format_v4i32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[VAR]]
;
%var = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0)
@@ -2093,7 +2093,7 @@ define float @extract0_bitcast_raw_ptr_buffer_load_format_v4i32(ptr addrspace(8)
define amdgpu_ps float @preserve_metadata_extract_elt0_raw_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -2514,7 +2514,7 @@ declare <4 x i8> @llvm.amdgcn.struct.buffer.load.v4i8(<4 x i32>, i32, i32, i32,
define amdgpu_ps float @struct_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2523,7 +2523,7 @@ define amdgpu_ps float @struct_ptr_buffer_load_f32(ptr addrspace(8) inreg %rsrc,
define amdgpu_ps <1 x float> @struct_ptr_buffer_load_v1f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.v1f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2532,7 +2532,7 @@ define amdgpu_ps <1 x float> @struct_ptr_buffer_load_v1f32(ptr addrspace(8) inre
define amdgpu_ps <2 x float> @struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2541,7 +2541,7 @@ define amdgpu_ps <2 x float> @struct_ptr_buffer_load_v2f32(ptr addrspace(8) inre
define amdgpu_ps <4 x float> @struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2550,7 +2550,7 @@ define amdgpu_ps <4 x float> @struct_ptr_buffer_load_v4f32(ptr addrspace(8) inre
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2561,7 +2561,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v2f32(ptr addrspace(
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v2f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2571,7 +2571,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v2f32(ptr addrspace(
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2582,7 +2582,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2593,7 +2593,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2604,7 +2604,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 12
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2614,7 +2614,7 @@ define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2625,7 +2625,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v4f32(ptr
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2636,7 +2636,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v4f32(ptr
define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2646,7 +2646,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2657,7 +2657,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_v4f3
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_ptr_buffer_load_v4f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2667,7 +2667,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_v4f3
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -2678,7 +2678,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_v4f3
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2689,7 +2689,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_v3f32(ptr addrspace(
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2700,7 +2700,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_v3f32(ptr addrspace(
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 8
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2710,7 +2710,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_v3f32(ptr addrspace(
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2721,7 +2721,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_v3f32(ptr
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_v3f32(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 4
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2731,7 +2731,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_v3f32(ptr
define i32 @extract0_bitcast_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_struct_ptr_buffer_load_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -2743,7 +2743,7 @@ define i32 @extract0_bitcast_struct_ptr_buffer_load_v4f32(ptr addrspace(8) inreg
define float @extract0_bitcast_struct_ptr_buffer_load_v4i32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_struct_ptr_buffer_load_v4i32(
-; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast i32 [[VAR]] to float
; CHECK-NEXT: ret float [[VAR2]]
;
@@ -2755,7 +2755,7 @@ define float @extract0_bitcast_struct_ptr_buffer_load_v4i32(ptr addrspace(8) inr
define amdgpu_ps float @preserve_metadata_extract_elt0_struct_ptr_buffer_load_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_ptr_buffer_load_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -2773,7 +2773,7 @@ declare <4 x i32> @llvm.amdgcn.struct.ptr.buffer.load.v4i32(ptr addrspace(8), i3
define amdgpu_ps half @extract_elt0_struct_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v2f16(
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2784,7 +2784,7 @@ define amdgpu_ps half @extract_elt0_struct_ptr_buffer_load_v2f16(ptr addrspace(8
define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v2f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2795,7 +2795,7 @@ define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v2f16(ptr addrspace(8
define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v3f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v3f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.v3f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2806,7 +2806,7 @@ define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v3f16(ptr addrspace(8
define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 2
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2817,7 +2817,7 @@ define amdgpu_ps half @extract_elt1_struct_ptr_buffer_load_v4f16(ptr addrspace(8
define amdgpu_ps half @extract_elt3_struct_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_v4f16(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 6
-; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call half @llvm.amdgcn.struct.ptr.buffer.load.f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret half [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2827,7 +2827,7 @@ define amdgpu_ps half @extract_elt3_struct_ptr_buffer_load_v4f16(ptr addrspace(8
define amdgpu_ps <2 x half> @extract_elt0_elt1_struct_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v4f16(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.load.v2f16(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x half> [[DATA]]
;
%data = call <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2842,7 +2842,7 @@ declare <4 x half> @llvm.amdgcn.struct.ptr.buffer.load.v4f16(ptr addrspace(8), i
define amdgpu_ps i8 @extract_elt0_struct_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_v2i8(
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2853,7 +2853,7 @@ define amdgpu_ps i8 @extract_elt0_struct_ptr_buffer_load_v2i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v2i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v2i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2864,7 +2864,7 @@ define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v2i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v3i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v3i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <3 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v3i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2875,7 +2875,7 @@ define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v3i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 1
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2886,7 +2886,7 @@ define amdgpu_ps i8 @extract_elt1_struct_ptr_buffer_load_v4i8(ptr addrspace(8) i
define amdgpu_ps i8 @extract_elt3_struct_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_v4i8(
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFS:%.*]], 3
-; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call i8 @llvm.amdgcn.struct.ptr.buffer.load.i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[TMP1]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret i8 [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -2896,7 +2896,7 @@ define amdgpu_ps i8 @extract_elt3_struct_ptr_buffer_load_v4i8(ptr addrspace(8) i
define amdgpu_ps <2 x i8> @extract_elt0_elt1_struct_ptr_buffer_load_v4i8(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_v4i8(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v2i8(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x i8> [[DATA]]
;
%data = call <4 x i8> @llvm.amdgcn.struct.ptr.buffer.load.v4i8(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3166,7 +3166,7 @@ declare <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32>, i32, i
define amdgpu_ps float @struct_ptr_buffer_load_format_f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3175,7 +3175,7 @@ define amdgpu_ps float @struct_ptr_buffer_load_format_f32(ptr addrspace(8) inreg
define amdgpu_ps <1 x float> @struct_ptr_buffer_load_format_v1f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_v1f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v1f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <1 x float> [[DATA]]
;
%data = call <1 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v1f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3184,7 +3184,7 @@ define amdgpu_ps <1 x float> @struct_ptr_buffer_load_format_v1f32(ptr addrspace(
define amdgpu_ps <2 x float> @struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3193,7 +3193,7 @@ define amdgpu_ps <2 x float> @struct_ptr_buffer_load_format_v2f32(ptr addrspace(
define amdgpu_ps <4 x float> @struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <4 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3202,7 +3202,7 @@ define amdgpu_ps <4 x float> @struct_ptr_buffer_load_format_v4f32(ptr addrspace(
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3212,7 +3212,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v2f32(ptr add
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3223,7 +3223,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v2f32(ptr add
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3233,7 +3233,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3244,7 +3244,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3255,7 +3255,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3266,7 +3266,7 @@ define amdgpu_ps float @extract_elt3_struct_ptr_buffer_load_format_v4f32(ptr add
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3276,7 +3276,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v4
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -3287,7 +3287,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v4
define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -3298,7 +3298,7 @@ define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_ptr_buffer_load_format_v4
define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <3 x float> [[DATA]]
;
%data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3308,7 +3308,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_ptr_buffer_load_form
define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -3319,7 +3319,7 @@ define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_ptr_buffer_load_form
define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
; CHECK-NEXT: ret <3 x float> [[SHUF]]
;
@@ -3330,7 +3330,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_ptr_buffer_load_form
define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3340,7 +3340,7 @@ define amdgpu_ps float @extract_elt0_struct_ptr_buffer_load_format_v3f32(ptr add
define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3351,7 +3351,7 @@ define amdgpu_ps float @extract_elt1_struct_ptr_buffer_load_format_v3f32(ptr add
define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt2_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
; CHECK-NEXT: ret float [[ELT1]]
;
@@ -3362,7 +3362,7 @@ define amdgpu_ps float @extract_elt2_struct_ptr_buffer_load_format_v3f32(ptr add
define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt0_elt1_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: ret <2 x float> [[DATA]]
;
%data = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
@@ -3372,7 +3372,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_ptr_buffer_load_format_v3
define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v3f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract_elt1_elt2_struct_ptr_buffer_load_format_v3f32(
-; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: ret <2 x float> [[SHUF]]
;
@@ -3383,7 +3383,7 @@ define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_ptr_buffer_load_format_v3
define i32 @extract0_bitcast_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @extract0_bitcast_struct_ptr_buffer_load_format_v4f32(
-; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0)
+; CHECK-NEXT: [[VAR:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]])
; CHECK-NEXT: [[VAR2:%.*]] = bitcast float [[VAR]] to i32
; CHECK-NEXT: ret i32 [[VAR2]]
;
@@ -3395,7 +3395,7 @@ define i32 @extract0_bitcast_struct_ptr_buffer_load_format_v4f32(ptr addrspace(8
define amdgpu_ps float @preserve_metadata_extract_elt0_struct_ptr_buffer_load_format_v2f32(ptr addrspace(8) inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_ptr_buffer_load_format_v2f32(
-; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0, metadata [[META1]]), !fpmath [[META0]]
; CHECK-NEXT: ret float [[DATA]]
;
%data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
diff --git a/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll b/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
index 65e8d85ca374a..5d42b18eb7f20 100644
--- a/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
+++ b/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
@@ -7,13 +7,13 @@ define void @hoistable_noalias(ptr addrspace(8) noalias %p, ptr addrspace(8) noa
; CHECK-LABEL: define void @hoistable_noalias
; CHECK-SAME: (ptr addrspace(8) noalias [[P:%.*]], ptr addrspace(8) noalias [[Q:%.*]], i32 [[BOUND:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0:![0-9]+]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -44,8 +44,8 @@ define void @not_hoistable_may_alias(ptr addrspace(8) %p, ptr addrspace(8) %q, i
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[INC:%.*]] = add i32 [[NOT_HOISTABLE]], [[ORIG]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
@@ -75,11 +75,11 @@ define void @hoistable_alias_scope(ptr addrspace(8) %p, ptr addrspace(8) %q, i32
; CHECK-LABEL: define void @hoistable_alias_scope
; CHECK-SAME: (ptr addrspace(8) [[P:%.*]], ptr addrspace(8) [[Q:%.*]], i32 [[BOUND:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META1:![0-9]+]], !noalias [[META4:![0-9]+]]
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0, metadata [[META0]]), !alias.scope [[META1:![0-9]+]], !noalias [[META4:![0-9]+]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META4]], !noalias [[META1]]
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
@@ -126,8 +126,8 @@ define void @not_hoistable_buffer_construction(ptr addrspace(1) noalias %p.globa
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0, metadata [[META0]])
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[INC:%.*]] = add i32 [[NOT_HOISTABLE]], [[ORIG]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
@@ -167,11 +167,11 @@ define void @hoistable_buffer_construction_intrinsic(ptr addrspace(1) noalias %p
; CHECK-NEXT: entry:
; CHECK-NEXT: [[P:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[P_GLOBAL]], i16 0, i64 0, i32 0)
; CHECK-NEXT: [[Q:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[Q_GLOBAL]], i16 0, i64 0, i32 0)
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
@@ -212,11 +212,11 @@ define void @hoistable_buffer_construction_alias_scope(ptr addrspace(1) %p.globa
; CHECK-NEXT: [[Q_EXT:%.*]] = zext i48 [[Q_TRUNC]] to i128
; CHECK-NEXT: [[P:%.*]] = inttoptr i128 [[P_EXT]] to ptr addrspace(8)
; CHECK-NEXT: [[Q:%.*]] = inttoptr i128 [[Q_EXT]] to ptr addrspace(8)
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META1]], !noalias [[META4]]
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0, metadata [[META0]]), !alias.scope [[META1]], !noalias [[META4]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META4]], !noalias [[META1]]
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index e67408a5e8c00..6943c90ec3c70 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -1989,9 +1989,10 @@ def ROCDL_RawPtrBufferLoadOp :
dag args = (ins Arg<ROCDLBufferRsrc, "", [MemRead]>:$rsrc,
I32:$offset,
I32:$soffset,
- ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux,
+ Arg<LLVM_MetadataType, "Atomicity of the access as !{ordering, syncscope}, or an empty metadata node if not atomic">:$atomicity);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($res)";
+ let assemblyFormat = "$rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $atomicity attr-dict `:` type($res)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2076,9 +2077,9 @@ def ROCDL_RawPtrBufferStoreOp :
I32:$offset,
I32:$soffset,
ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux,
- Arg<LLVM_MetadataType, "Sync scope for the access, or an empty metadata node if unspecified">:$syncscope);
+ Arg<LLVM_MetadataType, "Atomicity of the access as !{ordering, syncscope}, or an empty metadata node if not atomic">:$atomicity);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $syncscope attr-dict `:` type($vdata)";
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $atomicity attr-dict `:` type($vdata)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2096,9 +2097,9 @@ def ROCDL_RawPtrBufferAtomicCmpSwap :
I32:$offset,
I32:$soffset,
ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux,
- Arg<LLVM_MetadataType, "Sync scope for the access, or an empty metadata node if unspecified">:$syncscope);
+ Arg<LLVM_MetadataType, "Atomicity of the access as !{ordering, syncscope}, or an empty metadata node if not atomic">:$atomicity);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$src `,` $cmp `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $syncscope attr-dict `:` type($res)";
+ let assemblyFormat = "$src `,` $cmp `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $atomicity attr-dict `:` type($res)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2114,9 +2115,9 @@ class ROCDL_RawPtrBufferAtomicRet<string op> :
I32:$offset,
I32:$soffset,
ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux,
- Arg<LLVM_MetadataType, "Sync scope for the access, or an empty metadata node if unspecified">:$syncscope);
+ Arg<LLVM_MetadataType, "Atomicity of the access as !{ordering, syncscope}, or an empty metadata node if not atomic">:$atomicity);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $syncscope attr-dict `:` type($vdata)";
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) `,` $atomicity attr-dict `:` type($vdata)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 97aa2ad6508b2..2c6f49cfab20e 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -489,13 +489,9 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
args.push_back(sgprOffset);
- if constexpr (!std::is_same_v<Intrinsic, ROCDL::RawPtrBufferLoadOp>) {
- // No syncscope information is available at this level, so conservatively
- // report an empty (unspecified) scope.
- Value noScope = LLVM::MetadataAsValueOp::create(
- rewriter, loc, LLVM::MDNodeAttr::get(rewriter.getContext(), {}));
- args.push_back(noScope);
- }
+ // These ops are never atomic, so report the empty metadata node.
+ args.push_back(LLVM::MetadataAsValueOp::create(
+ rewriter, loc, LLVM::MDNodeAttr::get(rewriter.getContext(), {})));
llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
llvmBufferValType);
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
index 73e7e2e568204..3ce5473bf120f 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
@@ -130,7 +130,7 @@ func.func @gpu_gcn_raw_buffer_load_scalar_i32(%buf: memref<i32>) -> i32 {
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %[[stride]], %[[numRecords]], %[[flags]] : !llvm.ptr to <8>
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[] : memref<i32> -> i32
func.return %0 : i32
@@ -144,7 +144,7 @@ func.func @gpu_gcn_raw_buffer_load_i32(%buf: memref<64xi32>, %idx: i32) -> i32 {
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %[[stride]], %[[numRecords]], %[[flags]] : !llvm.ptr to <8>
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi32>, i32 -> i32
func.return %0 : i32
@@ -178,7 +178,7 @@ func.func @gpu_gcn_raw_buffer_load_i32_strided(%buf: memref<16x16xi32, strided<[
// CHECK: %[[vgpr_off:.*]] = llvm.mul %[[index]], %[[elem_size]] : i32
// CHECK: %[[zero_0:.*]] = llvm.mlir.constant(0 : i32) : i32
// CHECK: %[[sgpr_off:.*]] = llvm.mul %[[zero_0]], %[[elem_size]] : i32
- // CHECK: %[[v:.*]] = rocdl.raw.ptr.buffer.load %[[rsrc]], %[[vgpr_off]], %[[sgpr_off]], 0 : i32
+ // CHECK: %[[v:.*]] = rocdl.raw.ptr.buffer.load %[[rsrc]], %[[vgpr_off]], %[[sgpr_off]], 0, %{{.*}} : i32
// CHECK: return %[[v]] : i32
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%i, %j] : memref<16x16xi32, strided<[?, ?], offset: ?>>, i32, i32 -> i32
func.return %0 : i32
@@ -190,7 +190,7 @@ func.func @gpu_gcn_raw_buffer_load_i32_oob_off(%buf: memref<64xi32>, %idx: i32)
// RDNA: %[[flags:.*]] = llvm.mlir.constant(553807872 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %{{.*}}, %[[flags]]
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = false} %buf[%idx] : memref<64xi32>, i32 -> i32
func.return %0 : i32
@@ -198,7 +198,7 @@ func.func @gpu_gcn_raw_buffer_load_i32_oob_off(%buf: memref<64xi32>, %idx: i32)
// CHECK-LABEL: func @gpu_gcn_raw_buffer_load_1xi32
func.func @gpu_gcn_raw_buffer_load_1xi32(%buf: memref<64xi32>, %idx: i32) -> vector<1xi32> {
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: %[[cast:.*]] = llvm.bitcast %[[ret]] : i32 to vector<1xi32>
// CHECK: return %[[cast]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi32>, i32 -> vector<1xi32>
@@ -207,7 +207,7 @@ func.func @gpu_gcn_raw_buffer_load_1xi32(%buf: memref<64xi32>, %idx: i32) -> vec
// CHECK-LABEL: func @gpu_gcn_raw_buffer_load_2xi32
func.func @gpu_gcn_raw_buffer_load_2xi32(%buf: memref<64xi32>, %idx: i32) -> vector<2xi32> {
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xi32>
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<2xi32>
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi32>, i32 -> vector<2xi32>
func.return %0 : vector<2xi32>
@@ -217,7 +217,7 @@ func.func @gpu_gcn_raw_buffer_load_2xi32(%buf: memref<64xi32>, %idx: i32) -> vec
func.func @gpu_gcn_raw_buffer_load_i8(%buf: memref<64xi8>, %idx: i32) -> i8 {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i8
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i8
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi8>, i32 -> i8
func.return %0 : i8
@@ -227,7 +227,7 @@ func.func @gpu_gcn_raw_buffer_load_i8(%buf: memref<64xi8>, %idx: i32) -> i8 {
func.func @gpu_gcn_raw_buffer_load_2xi8(%buf: memref<64xi8>, %idx: i32) -> vector<2xi8> {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i16
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i16
// CHECK: %[[ret:.*]] = llvm.bitcast %[[loaded]] : i16 to vector<2xi8>
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi8>, i32 -> vector<2xi8>
@@ -236,7 +236,7 @@ func.func @gpu_gcn_raw_buffer_load_2xi8(%buf: memref<64xi8>, %idx: i32) -> vecto
// CHECK-LABEL: func @gpu_gcn_raw_buffer_load_16xi8
func.func @gpu_gcn_raw_buffer_load_16xi8(%buf: memref<64xi8>, %idx: i32) -> vector<16xi8> {
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xi32>
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<4xi32>
// CHECK: %[[ret:.*]] = llvm.bitcast %[[loaded]] : vector<4xi32> to vector<16xi8>
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi8>, i32 -> vector<16xi8>
@@ -247,7 +247,7 @@ func.func @gpu_gcn_raw_buffer_load_16xi8(%buf: memref<64xi8>, %idx: i32) -> vect
func.func @gpu_gcn_raw_buffer_load_f8E5M2FNUZ(%buf: memref<64xf8E5M2FNUZ>, %idx: i32) -> f8E5M2FNUZ {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i8
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i8
// CHECK: %[[ret:.*]] = builtin.unrealized_conversion_cast %[[loaded]] : i8 to f8E5M2FNUZ
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xf8E5M2FNUZ>, i32 -> f8E5M2FNUZ
@@ -258,7 +258,7 @@ func.func @gpu_gcn_raw_buffer_load_f8E5M2FNUZ(%buf: memref<64xf8E5M2FNUZ>, %idx:
func.func @gpu_gcn_raw_buffer_load_4xf8E4M3FNUZ(%buf: memref<64xf8E4M3FNUZ>, %idx: i32) -> vector<4xf8E4M3FNUZ> {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: %[[cast:.*]] = llvm.bitcast %[[loaded]] : i32 to vector<4xi8>
// CHECK: %[[ret:.*]] = builtin.unrealized_conversion_cast %[[cast]] : vector<4xi8> to vector<4xf8E4M3FNUZ>
// CHECK: return %[[ret]]
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index aac462ded97af..14480e3ba9920 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -884,9 +884,9 @@ llvm.func @rocdl.raw.ptr.buffer.f32(%rsrc : !llvm.ptr<8>,
%aux : i32, %vdata1 : f32,
%vdata2 : vector<2xf32>, %vdata4 : vector<4xf32>) {
// CHECK-LABEL: rocdl.raw.ptr.buffer.f32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<2xf32>
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<4xf32>
// CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
// CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : vector<2xf32>
@@ -895,18 +895,18 @@ llvm.func @rocdl.raw.ptr.buffer.f32(%rsrc : !llvm.ptr<8>,
// CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
// CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : f32
- %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
+ %atomicity = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
- %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : f32
- %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xf32>
- %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xf32>
+ %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0, %atomicity : f32
+ %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0, %atomicity : vector<2xf32>
+ %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0, %atomicity : vector<4xf32>
- rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
- rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0, %scope : vector<2xf32>
- rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, 0, %scope : vector<4xf32>
+ rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : f32
+ rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0, %atomicity : vector<2xf32>
+ rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, 0, %atomicity : vector<4xf32>
- %atomic_fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
- %atomic_fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
+ %atomic_fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : f32
+ %atomic_fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : f32
llvm.return
}
@@ -951,11 +951,11 @@ llvm.func @rocdl.raw.ptr.buffer.i32(%rsrc : !llvm.ptr<8>,
// CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
// CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.cmpswap %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0, %{{.*}} : i32
- %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
+ %atomicity = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
- %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
- %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
- %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
+ %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : i32
+ %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : i32
+ %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : i32
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 13fb3a1948abe..e6e8dab3750ee 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -1524,7 +1524,7 @@ llvm.func @rocdl.raw.ptr.buffer(%rsrc : !llvm.ptr<8>,
%vdata2 : vector<2xi32>,
%vdata4 : vector<4xi32>) {
%aux = llvm.mlir.constant(0 : i32) : i32
- %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
+ %atomicity = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer
// CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
@@ -1534,13 +1534,13 @@ llvm.func @rocdl.raw.ptr.buffer(%rsrc : !llvm.ptr<8>,
// CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
- %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : i32
- %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xi32>
- %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xi32>
+ %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0, %atomicity : i32
+ %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0, %atomicity : vector<2xi32>
+ %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0, %atomicity : vector<4xi32>
- rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
- rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0, %scope : vector<2xi32>
- rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %soffset, 0, %scope : vector<4xi32>
+ rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : i32
+ rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0, %atomicity : vector<2xi32>
+ rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %soffset, 0, %atomicity : vector<4xi32>
llvm.return
}
@@ -1691,14 +1691,14 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.f32(%rsrc : !llvm.ptr<8>,
%offset : i32, %soffset : i32,
%vdata1 : f32) -> f32 {
%aux = llvm.mlir.constant(0 : i32) : i32
- %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
+ %atomicity = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer.atomic.f32
// CHECK: [[PTR_FADD:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: [[PTR_FMAX:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: ret float [[PTR_FMAX]]
- %fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
- %fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0, %scope : f32
+ %fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : f32
+ %fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : f32
llvm.return %fmax : f32
}
@@ -1707,14 +1707,14 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.i32(%rsrc : !llvm.ptr<8>,
%offset : i32, %soffset : i32,
%vdata1 : i32) -> i32 {
%aux = llvm.mlir.constant(0 : i32) : i32
- %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
+ %atomicity = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer.atomic.i32
// CHECK: [[PTR_SMAX:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: [[PTR_UMIN:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: ret i32 [[PTR_UMIN]]
- %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
- %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0, %scope : i32
+ %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : i32
+ %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0, %atomicity : i32
llvm.return %umin : i32
}
@@ -1723,12 +1723,12 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.cmpswap(%rsrc : !llvm.ptr<8>,
%offset : i32, %soffset : i32,
%src : i32, %cmp : i32) -> i32 {
%aux = llvm.mlir.constant(0 : i32) : i32
- %scope = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
+ %atomicity = llvm.mlir.metadata_as_value #llvm.md_node<> : !llvm.metadata
// CHECK-LABEL: rocdl.raw.ptr.buffer.atomic.cmpswap
// CHECK: [[val:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %{{.*}}, i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}, metadata !{{[0-9]+}}
// CHECK: ret i32 [[val]]
- %val = rocdl.raw.ptr.buffer.atomic.cmpswap %src, %cmp, %rsrc, %offset, %soffset, 0, %scope : i32
+ %val = rocdl.raw.ptr.buffer.atomic.cmpswap %src, %cmp, %rsrc, %offset, %soffset, 0, %atomicity : i32
llvm.return %val : i32
}
More information about the cfe-commits
mailing list