[clang] [llvm] [AMDGPU] Fix missing cache bypass scope on seq_cst buffer fat pointer atomics (PR #213585)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 02:44:43 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/213585
>From 1d254511c60af3c1cb7575a5caa83635412a4915 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 31 Jul 2026 17:18:47 +0200
Subject: [PATCH 1/4] [AMDGPU] Fix missing cache bypass scope on seq_cst buffer
fat pointer atomics
Buffer atomic/load/store intrinsics have no scope operand, so the
scope was lost during lowering
Stash it as metadata and recover it in SIISelLowering so SIMemoryLegalizer sets cache bypass bits
---
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 16 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 27 +++
.../AMDGPU/memory-legalizer-buffer-atomics.ll | 155 +++++++++++++++++-
3 files changed, 196 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index dd2f7075c8df2..bb12289821749 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1566,6 +1566,10 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
+ // Buffer intrinsics have no ordering/scope operand, so stash it as
+ // metadata for SIISelLowering::getTgtMemIntrinsic to recover.
+ void attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
+ SyncScope::ID SSID);
Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
Align Alignment, AtomicOrdering Order,
bool IsVolatile, SyncScope::ID SSID);
@@ -1872,6 +1876,16 @@ void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A));
}
+void SplitPtrStructs::attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
+ SyncScope::ID SSID) {
+ if (Order == AtomicOrdering::NotAtomic)
+ return;
+ LLVMContext &Ctx = Call->getContext();
+ StringRef Scope = Ctx.getSyncScopeName(SSID).value_or("");
+ Call->setMetadata("amdgpu.buffer.atomic.scope",
+ MDNode::get(Ctx, MDString::get(Ctx, Scope)));
+}
+
void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
SyncScope::ID SSID) {
switch (Order) {
@@ -2023,6 +2037,7 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(IID, Ty, Args);
copyMetadata(Call, I);
+ attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, Alignment, Arg ? 1 : 0);
Call->takeName(I);
@@ -2093,6 +2108,7 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
IRB.getInt32(0), IRB.getInt32(Aux)});
copyMetadata(Call, &AI);
+ attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 403f5d7476dfc..dc9725a7c2de9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1416,6 +1416,16 @@ static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
return CI.getContext().getOrInsertSyncScopeID(Scope);
}
+// Recovers the scope AMDGPULowerBufferFatPointers stashed as metadata,
+// since buffer intrinsics have no scope operand of their own.
+static std::optional<SyncScope::ID> parseAtomicScopeMD(const CallBase &CI) {
+ MDNode *ScopeMD = CI.getMetadata("amdgpu.buffer.atomic.scope");
+ if (!ScopeMD)
+ return std::nullopt;
+ StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
+ return CI.getContext().getOrInsertSyncScopeID(Scope);
+}
+
void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
const CallBase &CI,
MachineFunction &MF,
@@ -1503,6 +1513,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = getValueType(MF.getDataLayout(), DataTy);
Info.flags = Flags | MachineMemOperand::MOStore;
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so SIMemoryLegalizer::expandStore sets cache
+ // bypass bits, since atomic and plain buffer stores look identical.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
} else {
// Atomic, NoReturn Sampler or prefetch
Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
@@ -1521,6 +1537,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// XXX - Should this be volatile without known ordering?
Info.flags |= MachineMemOperand::MOVolatile;
Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so expandAtomicCmpxchgOrRmw sets cache
+ // bypass bits.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
}
break;
case Intrinsic::amdgcn_raw_buffer_load_lds:
@@ -1561,6 +1583,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags = Flags | MachineMemOperand::MOLoad;
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so expandLoad sets cache bypass bits.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
Infos.push_back(Info);
return;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
index 1cf8ef7a918f4..6523c168dd50c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
@@ -22,7 +22,7 @@ define void @buffer_fat_ptr_agent_atomic_add_noret_i32(ptr addrspace(7) inreg %p
; GFX1250-NEXT: s_mov_b32 s3, s8
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_DEV
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%unused = atomicrmw add ptr addrspace(7) %ptr, i32 %val syncscope("agent") monotonic
ret void
@@ -49,13 +49,164 @@ define i32 @buffer_fat_ptr_agent_atomic_add_ret_i32(ptr addrspace(7) inreg %ptr,
; GFX1250-NEXT: s_mov_b32 s3, s8
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = atomicrmw add ptr addrspace(7) %ptr, i32 %val syncscope("agent") monotonic
ret i32 %ret
}
+define void @buffer_fat_ptr_seq_cst_atomic_add_noret_i32(ptr addrspace(7) inreg %ptr, i32 %val) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_atomic_add_noret_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %unused = atomicrmw add ptr addrspace(7) %ptr, i32 %val seq_cst
+ ret void
+}
+
+define i32 @buffer_fat_ptr_seq_cst_cmpxchg_i32(ptr addrspace(7) inreg %ptr, i32 %cmp, i32 %new) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_cmpxchg_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: scratch_store_b32 off, v1, s32 nv ; 4-byte Folded Spill
+; GFX1250-NEXT: v_mov_b32_e32 v2, v0
+; GFX1250-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_mov_b32_e32 v1, v2
+; GFX1250-NEXT: v_mov_b32_e32 v2, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %pair = cmpxchg ptr addrspace(7) %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+ %ret = extractvalue { i32, i1 } %pair, 0
+ ret i32 %ret
+}
+
+define i32 @buffer_fat_ptr_seq_cst_load_i32(ptr addrspace(7) inreg %ptr) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_load_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s16
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %ret = load atomic i32, ptr addrspace(7) %ptr seq_cst, align 4
+ ret i32 %ret
+}
+
+define void @buffer_fat_ptr_seq_cst_store_i32(ptr addrspace(7) inreg %ptr, i32 %val) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_store_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ store atomic i32 %val, ptr addrspace(7) %ptr seq_cst, align 4
+ ret void
+}
+
define void @raw_buffer_atomic_add_v2f16_noret(<2 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
; GFX1250-LABEL: raw_buffer_atomic_add_v2f16_noret:
; GFX1250: ; %bb.0:
>From 02209a898ab435f7c4f28c04aacb4e689a84f3bd Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 5 Aug 2026 06:15:48 +0200
Subject: [PATCH 2/4] doc
---
llvm/docs/AMDGPUUsage.rst | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 87ba811e1a686..a2f1d878cb783 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2484,6 +2484,28 @@ whether to skip the DPP optimization.
; Many expected active lanes: the DPP optimization is not skipped.
%old1 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 32}
+.. _amdgpu_buffer_atomic_scope:
+
+'``amdgpu.buffer.atomic.scope``' Metadata
+-------------------------------------------------
+
+Internal metadata attached by ``AMDGPULowerBufferFatPointers`` to the
+buffer intrinsics it emits for atomic, and ``seq_cst`` load or store,
+accesses through ptr addrspace(7) (buffer fat pointer). Buffer
+intrinsics have no scope operand of their own, so this metadata carries
+the :ref:`synchronization scope<amdgpu-memory-scopes>` name of the
+original memory operation through to instruction selection, where
+``SITargetLowering::getTgtMemIntrinsic`` recovers it so
+``SIMemoryLegalizer`` can set the correct cache bypass bits. This
+metadata is only produced and consumed internally by the AMDGPU backend
+and is not intended to be written by front ends.
+
+.. code-block:: llvm
+
+ %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0), !amdgpu.buffer.atomic.scope !0
+
+ !0 = !{!"agent"}
+
LLVM IR Attributes
==================
>From 0cdf82a3c04379316404a995fca772fb334a06a0 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 6 Aug 2026 10:55:19 +0200
Subject: [PATCH 3/4] address comment
---
llvm/docs/AMDGPUUsage.rst | 23 -
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 55 +-
llvm/lib/IR/AutoUpgrade.cpp | 157 +++
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 24 +-
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 44 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 33 +-
.../amdgpu-buffer-atomic-scope-upgrade.ll | 33 +
.../atomic_optimizations_mul_one.ll | 12 +-
.../AMDGPU/lower-buffer-fat-pointers-calls.ll | 4 +-
.../lower-buffer-fat-pointers-constants.ll | 4 +-
...ointers-contents-legalization-alignment.ll | 667 +++++-----
...ffer-fat-pointers-contents-legalization.ll | 255 ++--
.../lower-buffer-fat-pointers-mem-transfer.ll | 1089 +++++++++--------
.../lower-buffer-fat-pointers-memops.ll | 54 +-
...fer-fat-pointers-unoptimized-debug-data.ll | 4 +-
.../FunctionAttrs/make-buffer-rsrc.ll | 4 +-
.../LICM/AMDGPU/buffer-rsrc-ptrs.ll | 20 +-
17 files changed, 1372 insertions(+), 1110 deletions(-)
create mode 100644 llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index a2f1d878cb783..50caa5ba7e265 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2484,29 +2484,6 @@ whether to skip the DPP optimization.
; Many expected active lanes: the DPP optimization is not skipped.
%old1 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 32}
-.. _amdgpu_buffer_atomic_scope:
-
-'``amdgpu.buffer.atomic.scope``' Metadata
--------------------------------------------------
-
-Internal metadata attached by ``AMDGPULowerBufferFatPointers`` to the
-buffer intrinsics it emits for atomic, and ``seq_cst`` load or store,
-accesses through ptr addrspace(7) (buffer fat pointer). Buffer
-intrinsics have no scope operand of their own, so this metadata carries
-the :ref:`synchronization scope<amdgpu-memory-scopes>` name of the
-original memory operation through to instruction selection, where
-``SITargetLowering::getTgtMemIntrinsic`` recovers it so
-``SIMemoryLegalizer`` can set the correct cache bypass bits. This
-metadata is only produced and consumed internally by the AMDGPU backend
-and is not intended to be written by front ends.
-
-.. code-block:: llvm
-
- %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0), !amdgpu.buffer.atomic.scope !0
-
- !0 = !{!"agent"}
-
-
LLVM IR Attributes
==================
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 9c57b4d343fc6..b2862f1db0ea5 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1585,10 +1585,11 @@ class AMDGPURawPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
[AMDGPUBufferRsrcTy,// rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc,
+ llvm_i32_ty, // auxiliary data (imm, cachepolicy (bit 0 = glc,
// bit 1 = slc,
// bit 2 = dlc on gfx10+),
// swizzled buffer (bit 3 = swz))
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
def int_amdgcn_raw_ptr_atomic_buffer_load : AMDGPURawPtrAtomicBufferLoad;
@@ -1653,13 +1654,14 @@ class AMDGPUStructPtrAtomicBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsi
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // auxiliary/cachepolicy(imm):
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
// bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
// bit 3 = swz, bit 4 = scc (gfx90a)
// gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
// gfx12+: bits [0-2] = th, bits [3-4] = scope,
// bit 6 = swz
// all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<0>>,
ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<0>;
@@ -1700,7 +1702,23 @@ class AMDGPURawPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntr
ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1>;
def int_amdgcn_raw_ptr_buffer_store_format : AMDGPURawPtrBufferStore<llvm_anyfloat_ty>;
-def int_amdgcn_raw_ptr_buffer_store : AMDGPURawPtrBufferStore;
+def int_amdgcn_raw_ptr_buffer_store : DefaultAttrsIntrinsic <
+ [],
+ [llvm_any_ty, // vdata(VGPR)
+ AMDGPUBufferRsrcTy, // rsrc(SGPR)
+ llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
+ llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
+ // bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
+ // bit 3 = swz, bit 4 = scc (gfx90a)
+ // gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
+ // gfx12+: bits [0-2] = th, bits [3-4] = scope,
+ // bit 6 = swz
+ // all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
+ [IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
+ ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
+ AMDGPURsrcIntrinsic<1>;
class AMDGPUStructBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
[],
@@ -1739,7 +1757,24 @@ class AMDGPUStructPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsI
ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1>;
def int_amdgcn_struct_ptr_buffer_store_format : AMDGPUStructPtrBufferStore;
-def int_amdgcn_struct_ptr_buffer_store : AMDGPUStructPtrBufferStore;
+def int_amdgcn_struct_ptr_buffer_store : DefaultAttrsIntrinsic <
+ [],
+ [llvm_any_ty, // vdata(VGPR)
+ AMDGPUBufferRsrcTy, // rsrc(SGPR)
+ llvm_i32_ty, // vindex(VGPR)
+ llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
+ llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
+ llvm_i32_ty, // auxiliary/cachepolicy(imm):
+ // bit 0 = glc, bit 1 = slc, bit 2 = dlc (gfx10/gfx11),
+ // bit 3 = swz, bit 4 = scc (gfx90a)
+ // gfx942: bit 0 = sc0, bit 1 = nt, bit 3 = swz, bit 4 = sc1
+ // gfx12+: bits [0-2] = th, bits [3-4] = scope,
+ // bit 6 = swz
+ // all: volatile op (bit 31, stripped at lowering)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
+ [IntrArgMemOnly, IntrWriteMem, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
+ ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
+ AMDGPURsrcIntrinsic<1>;
class AMDGPURawBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
[data_ty],
@@ -1783,7 +1818,8 @@ class AMDGPURawPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1, 0>;
@@ -1811,7 +1847,8 @@ def int_amdgcn_raw_ptr_buffer_atomic_cmpswap : Intrinsic<
AMDGPUBufferRsrcTy, // rsrc(SGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<2>>,
ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<2, 0>;
@@ -1865,7 +1902,8 @@ class AMDGPUStructPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<1, 0>;
@@ -1891,7 +1929,8 @@ def int_amdgcn_struct_ptr_buffer_atomic_cmpswap : Intrinsic<
llvm_i32_ty, // vindex(VGPR)
llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling)
llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling)
- llvm_i32_ty], // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_i32_ty, // cachepolicy(imm; bit 1 = slc, ..., bit 31 = volatile)
+ llvm_metadata_ty], // syncscope name (empty = no scope info)
[IntrArgMemOnly, NoCapture<ArgIndex<2>>,
ImmArg<ArgIndex<6>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
AMDGPURsrcIntrinsic<2, 0>;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 4502759417c5a..865931fd1b1c0 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1400,6 +1400,88 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return true;
}
break;
+ // Legacy buffer atomic/store/atomic-load intrinsics missed the
+ // trailing syncscope metadata operand.
+ case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
+ if (F->arg_size() == 4) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
+ if (F->arg_size() == 5) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_raw_ptr_buffer_store:
+ if (F->arg_size() == 5) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_buffer_store:
+ if (F->arg_size() == 6) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
+ if (F->arg_size() == 5) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
+ if (F->arg_size() == 6) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
+ if (F->arg_size() == 6) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
+ if (F->arg_size() == 7) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
}
if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
@@ -4891,6 +4973,81 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
}
+ // Legacy buffer atomic/store/atomic-load intrinsics missed the trailing
+ // syncscope metadata operand. Append an empty MDNode, meaning "no scope
+ // info", matching the behavior of these calls before the operand existed.
+ auto UpgradeBufferScopeIntrinsicCall =
+ [](Function *F, CallBase *CI, IRBuilder<> &Builder,
+ ArrayRef<Type *> OverloadTys) -> Value * {
+ SmallVector<Value *, 8> Args(CI->args());
+ Args.push_back(MetadataAsValue::get(Builder.getContext(),
+ MDNode::get(Builder.getContext(), {})));
+
+ Function *NewDecl = Intrinsic::getOrInsertDeclaration(
+ F->getParent(), F->getIntrinsicID(), OverloadTys);
+
+ SmallVector<OperandBundleDef, 1> Bundles;
+ CI->getOperandBundlesAsDefs(Bundles);
+
+ auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
+ NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ NewCall->setCallingConv(CI->getCallingConv());
+ NewCall->setAttributes(CI->getAttributes());
+ NewCall->setDebugLoc(CI->getDebugLoc());
+ NewCall->copyMetadata(*CI);
+ NewCall->takeName(CI);
+ return NewCall;
+ };
+
+ switch (F->getIntrinsicID()) {
+ case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
+ case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
+ return UpgradeBufferScopeIntrinsicCall(F, CI, Builder,
+ {F->getReturnType()});
+ case Intrinsic::amdgcn_raw_ptr_buffer_store:
+ case Intrinsic::amdgcn_struct_ptr_buffer_store:
+ return UpgradeBufferScopeIntrinsicCall(F, CI, Builder,
+ {CI->getArgOperand(0)->getType()});
+ default:
+ break;
+ }
+
switch (F->getIntrinsicID()) {
default:
break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 6bf05f9808d78..3d115a1ed6551 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6690,11 +6690,17 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
unsigned ImmOffset;
+ // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // of the positional arguments counted below.
+ unsigned NumOperands = MI.getNumOperands();
+ if (MI.getOperand(NumOperands - 1).isMetadata())
+ --NumOperands;
+
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
// The struct intrinsic variants add one additional operand over raw.
- const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
+ const bool HasVIndex = NumOperands == NumVIndexOps;
Register VIndex;
int OpOffset = 0;
if (HasVIndex) {
@@ -6805,11 +6811,17 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
castBufferRsrcArgToV4I32(MI, B, 2 + OpOffset);
Register RSrc = MI.getOperand(2 + OpOffset).getReg();
+ // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // of the positional arguments counted below.
+ unsigned NumOperands = MI.getNumOperands();
+ if (MI.getOperand(NumOperands - 1).isMetadata())
+ --NumOperands;
+
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
// The struct intrinsic variants add one additional operand over raw.
- const bool HasVIndex = MI.getNumOperands() == NumVIndexOps + OpOffset;
+ const bool HasVIndex = NumOperands == NumVIndexOps + OpOffset;
Register VIndex;
if (HasVIndex) {
VIndex = MI.getOperand(3 + OpOffset).getReg();
@@ -7081,8 +7093,14 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
Register RSrc = MI.getOperand(3 + OpOffset).getReg();
const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
+ // The trailing syncscope metadata operand on _ptr_ intrinsics is not one
+ // of the positional arguments counted below.
+ unsigned NumOperands = MI.getNumOperands();
+ if (MI.getOperand(NumOperands - 1).isMetadata())
+ --NumOperands;
+
// The struct intrinsic variants add one additional operand over raw.
- const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
+ const bool HasVIndex = NumOperands == NumVIndexOps;
Register VIndex;
if (HasVIndex) {
VIndex = MI.getOperand(4 + OpOffset).getReg();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index bb12289821749..def10425dd165 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1566,10 +1566,9 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
- // Buffer intrinsics have no ordering/scope operand, so stash it as
- // metadata for SIISelLowering::getTgtMemIntrinsic to recover.
- void attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
- SyncScope::ID SSID);
+ // Build the trailing syncscope metadata argument for the buffer
+ // atomic/store/atomic-load intrinsics (empty string if not atomic).
+ Value *getScopeMDArg(AtomicOrdering Order, SyncScope::ID SSID);
Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
Align Alignment, AtomicOrdering Order,
bool IsVolatile, SyncScope::ID SSID);
@@ -1876,14 +1875,15 @@ void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A));
}
-void SplitPtrStructs::attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
- SyncScope::ID SSID) {
+Value *SplitPtrStructs::getScopeMDArg(AtomicOrdering Order,
+ SyncScope::ID SSID) {
+ LLVMContext &Ctx = IRB.getContext();
+ // An empty MDNode means not atomic. The system scope also has an empty
+ // name, so scope presence cannot be encoded by string emptiness alone.
if (Order == AtomicOrdering::NotAtomic)
- return;
- LLVMContext &Ctx = Call->getContext();
+ return MetadataAsValue::get(Ctx, MDNode::get(Ctx, {}));
StringRef Scope = Ctx.getSyncScopeName(SSID).value_or("");
- Call->setMetadata("amdgpu.buffer.atomic.scope",
- MDNode::get(Ctx, MDString::get(Ctx, Scope)));
+ return MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDString::get(Ctx, Scope)));
}
void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
@@ -1935,12 +1935,19 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
Aux |= AMDGPU::CPol::VOLATILE;
Args.push_back(IRB.getInt32(Aux));
+ // amdgcn_raw_ptr_buffer_load has no scope operand, unlike the atomic load,
+ // store, and RMW intrinsics used below.
+ bool NeedsScopeArg = true;
+
Intrinsic::ID IID = Intrinsic::not_intrinsic;
- if (isa<LoadInst>(I))
- IID = Order == AtomicOrdering::NotAtomic
- ? Intrinsic::amdgcn_raw_ptr_buffer_load
- : Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
- else if (isa<StoreInst>(I))
+ if (isa<LoadInst>(I)) {
+ if (Order == AtomicOrdering::NotAtomic) {
+ IID = Intrinsic::amdgcn_raw_ptr_buffer_load;
+ NeedsScopeArg = false;
+ } else {
+ IID = Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
+ }
+ } else if (isa<StoreInst>(I))
IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) {
switch (RMW->getOperation()) {
@@ -2035,9 +2042,11 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
}
}
+ if (NeedsScopeArg)
+ Args.push_back(getScopeMDArg(Order, SSID));
+
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(IID, Ty, Args);
copyMetadata(Call, I);
- attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, Alignment, Arg ? 1 : 0);
Call->takeName(I);
@@ -2106,9 +2115,8 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(
Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, Ty,
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
- IRB.getInt32(0), IRB.getInt32(Aux)});
+ IRB.getInt32(0), IRB.getInt32(Aux), getScopeMDArg(Order, SSID)});
copyMetadata(Call, &AI);
- attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index dc9725a7c2de9..69f50a966eaa3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1416,11 +1416,20 @@ static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
return CI.getContext().getOrInsertSyncScopeID(Scope);
}
-// Recovers the scope AMDGPULowerBufferFatPointers stashed as metadata,
-// since buffer intrinsics have no scope operand of their own.
-static std::optional<SyncScope::ID> parseAtomicScopeMD(const CallBase &CI) {
- MDNode *ScopeMD = CI.getMetadata("amdgpu.buffer.atomic.scope");
- if (!ScopeMD)
+// Buffer atomic/store/atomic-load intrinsics take the syncscope name as
+// their trailing metadata argument, an empty MDNode when the memory access
+// is not atomic (the system scope also has an empty name, so the MDNode
+// operand count, not string emptiness, distinguishes the two). Not every
+// intrinsic reaching this dispatch has that trailing argument, so callers
+// rely on this returning std::nullopt for those too.
+static std::optional<SyncScope::ID>
+parseBufferAtomicScopeArg(const CallBase &CI) {
+ auto *ScopeArg =
+ dyn_cast<MetadataAsValue>(CI.getArgOperand(CI.arg_size() - 1));
+ if (!ScopeArg)
+ return std::nullopt;
+ MDNode *ScopeMD = cast<MDNode>(ScopeArg->getMetadata());
+ if (ScopeMD->getNumOperands() == 0)
return std::nullopt;
StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
return CI.getContext().getOrInsertSyncScopeID(Scope);
@@ -1447,7 +1456,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
if (!IsSPrefetch) {
- auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
+ // Some buffer intrinsics have a trailing syncscope metadata arg after
+ // the cachepolicy immarg.
+ unsigned AuxIdx = CI.arg_size() - 1;
+ if (isa<MetadataAsValue>(CI.getArgOperand(AuxIdx)))
+ --AuxIdx;
+ auto *Aux = cast<ConstantInt>(CI.getArgOperand(AuxIdx));
if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
Flags |= MachineMemOperand::MOVolatile;
}
@@ -1513,7 +1527,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = getValueType(MF.getDataLayout(), DataTy);
Info.flags = Flags | MachineMemOperand::MOStore;
- if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ if (std::optional<SyncScope::ID> SSID = parseBufferAtomicScopeArg(CI)) {
// Pretend to be atomic so SIMemoryLegalizer::expandStore sets cache
// bypass bits, since atomic and plain buffer stores look identical.
Info.order = AtomicOrdering::Monotonic;
@@ -1537,7 +1551,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// XXX - Should this be volatile without known ordering?
Info.flags |= MachineMemOperand::MOVolatile;
Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
- if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ if (std::optional<SyncScope::ID> SSID =
+ parseBufferAtomicScopeArg(CI)) {
// Pretend to be atomic so expandAtomicCmpxchgOrRmw sets cache
// bypass bits.
Info.order = AtomicOrdering::Monotonic;
@@ -1583,7 +1598,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags = Flags | MachineMemOperand::MOLoad;
- if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ if (std::optional<SyncScope::ID> SSID = parseBufferAtomicScopeArg(CI)) {
// Pretend to be atomic so expandLoad sets cache bypass bits.
Info.order = AtomicOrdering::Monotonic;
Info.ssid = *SSID;
diff --git a/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll b/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
new file mode 100644
index 0000000000000..3f9550e026c8e
--- /dev/null
+++ b/llvm/test/Bitcode/amdgpu-buffer-atomic-scope-upgrade.ll
@@ -0,0 +1,33 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+declare void @llvm.amdgcn.raw.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32, i32, ptr addrspace(8), i32, i32, i32 immarg)
+declare void @llvm.amdgcn.struct.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32, i32, ptr addrspace(8), i32, i32, i32, i32 immarg)
+
+define void @test(i32 %v, ptr addrspace(8) %rsrc, float %f) {
+; CHECK-LABEL: define void @test(
+; CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+; CHECK: call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0, metadata !0)
+ call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %a = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %b = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ %c = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %f, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %d = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %e = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %g = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i32(i32 %v, i32 %v, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+; CHECK: !0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index 4d08b2c582d2b..7fc4b3fbd9a50 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -322,7 +322,7 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: ret void
@@ -362,7 +362,7 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
@@ -422,7 +422,7 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: ret void
@@ -462,7 +462,7 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
@@ -523,7 +523,7 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB12]]
; IR: [[BB12]]:
; IR-NEXT: ret void
@@ -565,7 +565,7 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[TMP10:.*]], label %[[BB12:.*]]
; IR: [[TMP10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0, metadata !0)
; IR-NEXT: br label %[[BB12]]
; IR: [[BB12]]:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP11]], %[[TMP10]] ]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
index cf346a8fa2064..12cdaf3536f78 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
@@ -91,10 +91,10 @@ define void @caller(ptr addrspace(7) noundef nonnull %arg) {
; CHECK-NEXT: [[V_INT:%.*]] = or i160 [[TMP1]], [[V_INT_OFF]]
; CHECK-NEXT: [[V_INT_LEGAL:%.*]] = bitcast i160 [[V_INT]] to <5 x i32>
; CHECK-NEXT: [[V_INT_SLICE_0:%.*]] = shufflevector <5 x i32> [[V_INT_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[V_INT_SLICE_0]], ptr addrspace(8) align 32 [[ARG_RSRC]], i32 [[ARG_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[V_INT_SLICE_0]], ptr addrspace(8) align 32 [[ARG_RSRC]], i32 [[ARG_OFF]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[ARG_PART_4:%.*]] = add i32 [[ARG_OFF]], 16
; CHECK-NEXT: [[V_INT_SLICE_4:%.*]] = extractelement <5 x i32> [[V_INT_LEGAL]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[V_INT_SLICE_4]], ptr addrspace(8) align 16 [[ARG_RSRC]], i32 [[ARG_PART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[V_INT_SLICE_4]], ptr addrspace(8) align 16 [[ARG_RSRC]], i32 [[ARG_PART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
%v = call ptr addrspace(7) @extern(ptr addrspace(7) %arg)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
index e471178394303..9a94a12060f2d 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
@@ -212,7 +212,7 @@ define i32 @load_null() {
define void @store_null() {
; CHECK-LABEL: define void @store_null() {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: ret void
;
store i32 0, ptr addrspace(7) null, align 4
@@ -230,7 +230,7 @@ define i32 @load_poison() {
define void @store_poison() {
; CHECK-LABEL: define void @store_poison() {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
store i32 0, ptr addrspace(7) poison, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
index 95d0ed9114731..a221056406da2 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
@@ -37,22 +37,22 @@ define i32 @load_i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_i32_align4(i32 %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_i32_align4(
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align4(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align4(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align4(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1:![0-9]+]])
; BOTH_FLAGS-NEXT: ret void
;
@@ -104,30 +104,30 @@ define void @store_i32_align2(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <2 x i16>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align2(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align2(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <2 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align2(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -190,42 +190,42 @@ define void @store_i32_align1(i32 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <4 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i32_align1(
; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align1(
; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <4 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i32_align1(
; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -264,22 +264,22 @@ define i64 @load_i64_align4(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_i64_align4(i64 %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_i64_align4(
; STRICT-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i64_align4(
; UNALIGNED_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i64_align4(
; RELAXED_OOB_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i64_align4(
; BOTH_FLAGS-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -366,66 +366,66 @@ define void @store_i64_align1(i64 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i64 [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i64_align1(
; UNALIGNED_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i64_align1(
; RELAXED_OOB_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i64 [[DATA]] to <8 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i64_align1(
; BOTH_FLAGS-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -473,29 +473,29 @@ define void @store_v2i32_align4(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v2i32_align4(
; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align4(
; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align4(
; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i32_align4(
; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -562,46 +562,46 @@ define void @store_v2i32_align2(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <4 x i16>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align2(
; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align2(
; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <4 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i32_align2(
; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -692,70 +692,70 @@ define void @store_v2i32_align1(<2 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align1(
; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align1(
; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <8 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i32_align1(
; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -809,35 +809,35 @@ define void @store_v3i32_align4(<3 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v3i32_align4(
; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <3 x i32> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <3 x i32> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align4(
; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <3 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <3 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align4(
; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v3i32_align4(
; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -887,29 +887,29 @@ define void @store_v3i32_align8(<3 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v3i32_align8(
; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i32> [[DATA]], <3 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align8(
; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i32> [[DATA]], <3 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align8(
; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v3i32_align8(
; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -948,22 +948,22 @@ define <3 x i32> @load_v3i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
define void @store_v3i32_align16(<3 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
; STRICT-LABEL: define void @store_v3i32_align16(
; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align16(
; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align16(
; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v3i32_align16(
; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1015,29 +1015,29 @@ define void @store_v4i32_align8(<4 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v4i32_align8(
; STRICT-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i32_align8(
; UNALIGNED_ONLY-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i32_align8(
; RELAXED_OOB_ONLY-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i32_align8(
; BOTH_FLAGS-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1101,37 +1101,37 @@ define void @store_v8i32_align16(<8 x i32> %data, ptr addrspace(8) inreg %buf, i
; STRICT-LABEL: define void @store_v8i32_align16(
; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align16(
; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align16(
; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v8i32_align16(
; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1211,49 +1211,49 @@ define void @store_v8i32_align8(<8 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v8i32_align8(
; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align8(
; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align8(
; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v8i32_align8(
; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1349,73 +1349,73 @@ define void @store_v8i32_align4(<8 x i32> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v8i32_align4(
; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i32> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i32> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i32> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 12
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i32> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i32> [[DATA]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 20
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i32> [[DATA]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i32> [[DATA]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 28
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i32> [[DATA]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align4(
; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i32> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i32> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i32> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 12
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i32> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i32> [[DATA]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 20
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i32> [[DATA]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i32> [[DATA]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 28
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i32> [[DATA]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align4(
; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v8i32_align4(
; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1495,49 +1495,49 @@ define void @store_v10i32_align16(<10 x i32> %data, ptr addrspace(8) inreg %buf,
; STRICT-LABEL: define void @store_v10i32_align16(
; STRICT-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; STRICT-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v10i32_align16(
; UNALIGNED_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v10i32_align16(
; RELAXED_OOB_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v10i32_align16(
; BOTH_FLAGS-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; BOTH_FLAGS-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1633,61 +1633,61 @@ define void @store_v10i32_align8(<10 x i32> %data, ptr addrspace(8) inreg %buf,
; STRICT-LABEL: define void @store_v10i32_align8(
; STRICT-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; STRICT-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v10i32_align8(
; UNALIGNED_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v10i32_align8(
; RELAXED_OOB_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v10i32_align8(
; BOTH_FLAGS-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
; BOTH_FLAGS-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1738,30 +1738,30 @@ define void @store_i16_align1(i16 %data, ptr addrspace(8) inreg %buf, i32 %off)
; STRICT-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i16 [[DATA]] to <2 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_i16_align1(
; UNALIGNED_ONLY-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_i16_align1(
; RELAXED_OOB_ONLY-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i16 [[DATA]] to <2 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_i16_align1(
; BOTH_FLAGS-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1809,29 +1809,29 @@ define void @store_v2i16_align4(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v2i16_align4(
; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align4(
; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align4(
; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i16_align4(
; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1883,33 +1883,33 @@ define void @store_v2i16_align2(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-LABEL: define void @store_v2i16_align2(
; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align2(
; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align2(
; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i16_align2(
; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1976,46 +1976,46 @@ define void @store_v2i16_align1(<2 x i16> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i16> [[DATA]] to <4 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align1(
; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align1(
; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i16> [[DATA]] to <4 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i16_align1(
; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2065,31 +2065,31 @@ define void @store_v2i8_align2(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v2i8_align2(
; STRICT-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i8_align2(
; UNALIGNED_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i8_align2(
; RELAXED_OOB_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i8_align2(
; BOTH_FLAGS-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2142,34 +2142,34 @@ define void @store_v2i8_align1(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v2i8_align1(
; STRICT-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v2i8_align1(
; UNALIGNED_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v2i8_align1(
; RELAXED_OOB_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v2i8_align1(
; BOTH_FLAGS-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2231,43 +2231,43 @@ define void @store_v4i8_align4(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v4i8_align4(
; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align4(
; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align4(
; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i8_align4(
; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2333,47 +2333,47 @@ define void @store_v4i8_align2(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v4i8_align2(
; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align2(
; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align2(
; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to <2 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i8_align2(
; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2444,52 +2444,52 @@ define void @store_v4i8_align1(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32
; STRICT-LABEL: define void @store_v4i8_align1(
; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align1(
; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align1(
; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v4i8_align1(
; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2578,68 +2578,68 @@ define void @store_v16i4_align8(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align8(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align8(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align8(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2728,68 +2728,68 @@ define void @store_v16i4_align4(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align4(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align4(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align4(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2888,78 +2888,78 @@ define void @store_v16i4_align2(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align2(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align2(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <4 x i16>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align2(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3070,90 +3070,90 @@ define void @store_v16i4_align1(<16 x i4> %data, ptr addrspace(8) inreg %buf, i3
; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; STRICT-NEXT: ret void
;
; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align1(
; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; UNALIGNED_ONLY-NEXT: ret void
;
; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align1(
; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) {
; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
-; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0, metadata [[META1]])
; RELAXED_OOB_ONLY-NEXT: ret void
;
; BOTH_FLAGS-LABEL: define void @store_v16i4_align1(
; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
-; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0, metadata [[META1]])
; BOTH_FLAGS-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3278,3 +3278,12 @@ define <8 x half> @load_v8f16_align2_not_oob(ptr addrspace(1) inreg %ptr, i32 %o
!llvm.module.flags = !{!0}
!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 BUFFER_OOB_MODE}
+;.
+; STRICT: [[META1]] = !{}
+;.
+; UNALIGNED_ONLY: [[META1]] = !{}
+;.
+; RELAXED_OOB_ONLY: [[META1]] = !{}
+;.
+; BOTH_FLAGS: [[META1]] = !{}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
index 5efed9e569f2f..03fef762d8818 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
@@ -23,7 +23,7 @@ define i8 @load_i8(ptr addrspace(8) inreg %buf) {
define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i8(
; CHECK-SAME: i8 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1:![0-9]+]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -45,7 +45,7 @@ define i16 @load_i16(ptr addrspace(8) inreg %buf) {
define void @store_i16(i16 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i16(
; CHECK-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -67,7 +67,7 @@ define i32 @load_i32(ptr addrspace(8) inreg %buf) {
define void @store_i32(i32 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i32(
; CHECK-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -89,7 +89,7 @@ define i64 @load_i64(ptr addrspace(8) inreg %buf) {
define void @store_i64(i64 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i64(
; CHECK-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -111,7 +111,7 @@ define i128 @load_i128(ptr addrspace(8) inreg %buf) {
define void @store_i128(i128 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i128(
; CHECK-SAME: i128 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i128(i128 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i128(i128 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -135,7 +135,7 @@ define void @store_v1i32(<1 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i32(
; CHECK-SAME: <1 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -157,7 +157,7 @@ define <2 x i32> @load_v2i32(ptr addrspace(8) inreg %buf) {
define void @store_v2i32(<2 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i32(
; CHECK-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -179,7 +179,7 @@ define <3 x i32> @load_v3i32(ptr addrspace(8) inreg %buf) {
define void @store_v3i32(<3 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i32(
; CHECK-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -201,7 +201,7 @@ define <4 x i32> @load_v4i32(ptr addrspace(8) inreg %buf) {
define void @store_v4i32(<4 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i32(
; CHECK-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -223,7 +223,7 @@ define <2 x i16> @load_v2i16(ptr addrspace(8) inreg %buf) {
define void @store_v2i16(<2 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i16(
; CHECK-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -245,7 +245,7 @@ define <4 x i16> @load_v4i16(ptr addrspace(8) inreg %buf) {
define void @store_v4i16(<4 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i16(
; CHECK-SAME: <4 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -267,7 +267,7 @@ define <8 x i16> @load_v8i16(ptr addrspace(8) inreg %buf) {
define void @store_v8i16(<8 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i16(
; CHECK-SAME: <8 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -289,7 +289,7 @@ define <2 x i64> @load_v2i64(ptr addrspace(8) inreg %buf) {
define void @store_v2i64(<2 x i64> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i64(
; CHECK-SAME: <2 x i64> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i64(<2 x i64> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i64(<2 x i64> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -311,7 +311,7 @@ define half @load_f16(ptr addrspace(8) inreg %buf) {
define void @store_f16(half %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_f16(
; CHECK-SAME: half [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -333,7 +333,7 @@ define bfloat @load_bf16(ptr addrspace(8) inreg %buf) {
define void @store_bf16(bfloat %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_bf16(
; CHECK-SAME: bfloat [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -355,7 +355,7 @@ define <2 x half> @load_v2f16(ptr addrspace(8) inreg %buf) {
define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f16(
; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -381,9 +381,9 @@ define void @store_v2f16_align2(<2 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f16_align2(
; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x half> [[DATA]], i64 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x half> [[DATA]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -405,7 +405,7 @@ define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {
define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4bf16(
; CHECK-SAME: <4 x bfloat> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -427,7 +427,7 @@ define <8 x half> @load_v8f16(ptr addrspace(8) inreg %buf) {
define void @store_v8f16(<8 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8f16(
; CHECK-SAME: <8 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8f16(<8 x half> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8f16(<8 x half> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -449,7 +449,7 @@ define float @load_f32(ptr addrspace(8) inreg %buf) {
define void @store_f32(float %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_f32(
; CHECK-SAME: float [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -471,7 +471,7 @@ define <2 x float> @load_v2f32(ptr addrspace(8) inreg %buf) {
define void @store_v2f32(<2 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f32(
; CHECK-SAME: <2 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -493,7 +493,7 @@ define <3 x float> @load_v3f32(ptr addrspace(8) inreg %buf) {
define void @store_v3f32(<3 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3f32(
; CHECK-SAME: <3 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -515,7 +515,7 @@ define <4 x float> @load_v4f32(ptr addrspace(8) inreg %buf) {
define void @store_v4f32(<4 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4f32(
; CHECK-SAME: <4 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -537,7 +537,7 @@ define ptr addrspace(0) @load_p0(ptr addrspace(8) inreg %buf) {
define void @store_p0(ptr addrspace(0) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p0(
; CHECK-SAME: ptr [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -559,7 +559,7 @@ define ptr addrspace(1) @load_p1(ptr addrspace(8) inreg %buf) {
define void @store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p1(
; CHECK-SAME: ptr addrspace(1) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -581,7 +581,7 @@ define ptr addrspace(2) @load_p2(ptr addrspace(8) inreg %buf) {
define void @store_p2(ptr addrspace(2) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p2(
; CHECK-SAME: ptr addrspace(2) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p2(ptr addrspace(2) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p2(ptr addrspace(2) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -603,7 +603,7 @@ define ptr addrspace(3) @load_p3(ptr addrspace(8) inreg %buf) {
define void @store_p3(ptr addrspace(3) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p3(
; CHECK-SAME: ptr addrspace(3) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p3(ptr addrspace(3) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p3(ptr addrspace(3) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -625,7 +625,7 @@ define ptr addrspace(4) @load_p4(ptr addrspace(8) inreg %buf) {
define void @store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p4(
; CHECK-SAME: ptr addrspace(4) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -647,7 +647,7 @@ define ptr addrspace(5) @load_p5(ptr addrspace(8) inreg %buf) {
define void @store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p5(
; CHECK-SAME: ptr addrspace(5) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -669,7 +669,7 @@ define ptr addrspace(6) @load_p6(ptr addrspace(8) inreg %buf) {
define void @store_p6(ptr addrspace(6) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p6(
; CHECK-SAME: ptr addrspace(6) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p6(ptr addrspace(6) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p6(ptr addrspace(6) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -691,7 +691,7 @@ define ptr addrspace(8) @load_p8(ptr addrspace(8) inreg %buf) {
define void @store_p8(ptr addrspace(8) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p8(
; CHECK-SAME: ptr addrspace(8) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p8(ptr addrspace(8) [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.p8(ptr addrspace(8) [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -713,7 +713,7 @@ define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(8) inreg %buf) {
define void @store_v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2p1(
; CHECK-SAME: <2 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -735,7 +735,7 @@ define <2 x ptr addrspace(5)> @load_v2p5(ptr addrspace(8) inreg %buf) {
define void @store_v2p5(<2 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2p5(
; CHECK-SAME: <2 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p5(<2 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p5(<2 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -757,7 +757,7 @@ define <3 x ptr addrspace(5)> @load_v3p5(ptr addrspace(8) inreg %buf) {
define void @store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3p5(
; CHECK-SAME: <3 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -779,7 +779,7 @@ define <4 x ptr addrspace(5)> @load_v4p5(ptr addrspace(8) inreg %buf) {
define void @store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4p5(
; CHECK-SAME: <4 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -805,7 +805,7 @@ define void @store_v6f16(<6 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6f16(
; CHECK-SAME: <6 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <6 x half> [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -834,9 +834,9 @@ define void @store_v5f32(<5 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5f32(
; CHECK-SAME: <5 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x float> [[DATA]], <5 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x float> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -864,9 +864,9 @@ define void @store_v6f32(<6 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6f32(
; CHECK-SAME: <6 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -894,9 +894,9 @@ define void @store_v7f32(<7 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v7f32(
; CHECK-SAME: <7 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <3 x i32> <i32 4, i32 5, i32 6>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -924,9 +924,9 @@ define void @store_v8f32(<8 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8f32(
; CHECK-SAME: <8 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -957,11 +957,11 @@ define void @store_v10f32(<10 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v10f32(
; CHECK-SAME: <10 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <2 x i32> <i32 8, i32 9>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_8]], ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_8]], ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -989,9 +989,9 @@ define void @store_v6i32(<6 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i32(
; CHECK-SAME: <6 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1019,9 +1019,9 @@ define void @store_v4p1(<4 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %bu
; CHECK-LABEL: define void @store_v4p1(
; CHECK-SAME: <4 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_2]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_2]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1047,7 +1047,7 @@ define void @store_v1i16(<1 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i16(
; CHECK-SAME: <1 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <1 x i16> [[DATA]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_STORABLE]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_STORABLE]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1074,9 +1074,9 @@ define void @store_v3i16(<3 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i16(
; CHECK-SAME: <3 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1103,9 +1103,9 @@ define void @store_v5i16(<5 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5i16(
; CHECK-SAME: <5 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i16> [[DATA]], <5 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i16> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1129,7 +1129,7 @@ define void @store_v6i16(<6 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i16(
; CHECK-SAME: <6 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <6 x i16> [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1158,9 +1158,9 @@ define void @store_v7i16(<7 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <7 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <7 x i16> [[DATA]], <7 x i16> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <6 x i16> [[DATA_SLICE_0]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <7 x i16> [[DATA]], i64 6
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1187,9 +1187,9 @@ define void @store_v9i16(<9 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v9i16(
; CHECK-SAME: <9 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <9 x i16> [[DATA]], <9 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_8:%.*]] = extractelement <9 x i16> [[DATA]], i64 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1217,7 +1217,7 @@ define void @store_v1i8(<1 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i8(
; CHECK-SAME: <1 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <1 x i8> [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1241,7 +1241,7 @@ define void @store_v2i8(<2 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i8(
; CHECK-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1270,9 +1270,9 @@ define void @store_v3i8(<3 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <3 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i8> [[DATA]], <3 x i8> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_0]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i8> [[DATA]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1296,7 +1296,7 @@ define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i8(
; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1325,9 +1325,9 @@ define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <5 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1356,9 +1356,9 @@ define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1391,12 +1391,12 @@ define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <7 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[DATA_SLICE_4_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_4]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <7 x i8> [[DATA]], i64 6
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1420,7 +1420,7 @@ define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i8(
; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <8 x i8> [[DATA]] to <2 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1458,21 +1458,21 @@ define void @store_v8i8_align1(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i8_align1(
; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA]], i64 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA]], i64 5
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA]], i64 6
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA]], i64 7
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1496,7 +1496,7 @@ define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v12i8(
; CHECK-SAME: <12 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <12 x i8> [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1520,7 +1520,7 @@ define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v16i8(
; CHECK-SAME: <16 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i8> [[DATA]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1550,9 +1550,9 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <32 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <32 x i8> [[DATA]] to <8 x i32>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1582,7 +1582,7 @@ define void @store_a1i32([1 x i32] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_ELEM_0:%.*]] = extractvalue [1 x i32] [[DATA]], 0
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <1 x i32> poison, i32 [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA_AS_VEC_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1612,7 +1612,7 @@ define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA_AS_VEC_0]], i32 [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1643,8 +1643,8 @@ define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x half> poison, half [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x half] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x half> [[DATA_AS_VEC_0]], half [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1674,7 +1674,7 @@ define void @store_a2p1([2 x ptr addrspace(1)] %data, ptr addrspace(8) inreg %bu
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x ptr addrspace(1)> poison, ptr addrspace(1) [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x ptr addrspace(1)] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x ptr addrspace(1)> [[DATA_AS_VEC_0]], ptr addrspace(1) [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_AS_VEC_1]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_AS_VEC_1]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1707,9 +1707,9 @@ define void @store_i40(i40 %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i40 [[DATA]] to <5 x i8>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA_LEGAL]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA_LEGAL]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1733,7 +1733,7 @@ define void @store_i96(i96 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i96(
; CHECK-SAME: i96 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i96 [[DATA]] to <3 x i32>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1762,9 +1762,9 @@ define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: i160 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i160 [[DATA]] to <5 x i32>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <5 x i32> [[DATA_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <5 x i32> [[DATA_LEGAL]], i64 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1794,9 +1794,9 @@ define void @store_i256(i256 %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: i256 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast i256 [[DATA]] to <8 x i32>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1822,7 +1822,7 @@ define void @store_i7(i7 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i7(
; CHECK-SAME: i7 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_ZEXT:%.*]] = zext i7 [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1846,7 +1846,7 @@ define void @store_i4(i4 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i4(
; CHECK-SAME: i4 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_ZEXT:%.*]] = zext i4 [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1872,7 +1872,7 @@ define void @store_v2i4(<2 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i4(
; CHECK-SAME: <2 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i4> [[DATA]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1896,7 +1896,7 @@ define void @store_v4i4(<4 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i4(
; CHECK-SAME: <4 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i4> [[DATA]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1920,7 +1920,7 @@ define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i4(
; CHECK-SAME: <8 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <8 x i4> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1948,7 +1948,7 @@ define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <2 x i6> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_AS_SCALAR:%.*]] = bitcast <2 x i6> [[DATA]] to i12
; CHECK-NEXT: [[DATA_ZEXT:%.*]] = zext i12 [[DATA_AS_SCALAR]] to i16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_ZEXT]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_ZEXT]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1981,9 +1981,9 @@ define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i32> [[DATA_ABI:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA:%.*]] = bitcast <6 x i32> [[DATA_ABI]] to <32 x i6>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%data = bitcast <6 x i32> %data.abi to <32 x i6>
@@ -2010,7 +2010,7 @@ define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @volatile_store_v4i8(
; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2039,9 +2039,9 @@ define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 2
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2079,13 +2079,13 @@ define void @store_a2a2i32([2 x [2 x i32]] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA0_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA0_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA0_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA0]], 1
; CHECK-NEXT: [[DATA0_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA0_AS_VEC_0]], i32 [[DATA0_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [2 x [2 x i32]] [[DATA]], 1
; CHECK-NEXT: [[DATA1_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA1]], 0
; CHECK-NEXT: [[DATA1_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA1_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA1_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA1]], 1
; CHECK-NEXT: [[DATA1_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA1_AS_VEC_0]], i32 [[DATA1_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2111,9 +2111,9 @@ define void @store_a2v2i32([2 x <2 x i32>] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2v2i32(
; CHECK-SAME: [2 x <2 x i32>] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA0:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2139,9 +2139,9 @@ define void @store_a2v3i32([2 x <3 x i32>] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2v3i32(
; CHECK-SAME: [2 x <3 x i32>] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA0:%.*]] = extractvalue [2 x <3 x i32>] [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [2 x <3 x i32>] [[DATA]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA1]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA1]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2165,7 +2165,7 @@ define void @store_sl_i32s({ i32 } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_i32s(
; CHECK-SAME: { i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0:%.*]] = extractvalue { i32 } [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2189,7 +2189,7 @@ define void @store_sl_sl_f32ss({ { float } } %data, ptr addrspace(8) inreg %buf)
; CHECK-LABEL: define void @store_sl_sl_f32ss(
; CHECK-SAME: { { float } } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0_0:%.*]] = extractvalue { { float } } [[DATA]], 0, 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_0_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_0_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2213,7 +2213,7 @@ define void @store_sl_v2i32s({ <2 x i32> } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_v2i32s(
; CHECK-SAME: { <2 x i32> } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0:%.*]] = extractvalue { <2 x i32> } [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2239,9 +2239,9 @@ define void @store_sl_i64i32s({ i64, i32 } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_i64i32s(
; CHECK-SAME: { i64, i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA_0:%.*]] = extractvalue { i64, i32 } [[DATA]], 0
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA_1:%.*]] = extractvalue { i64, i32 } [[DATA]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2276,16 +2276,16 @@ define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: [4 x i7] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
; CHECK-NEXT: [[DATA0:%.*]] = extractvalue [4 x i7] [[DATA]], 0
; CHECK-NEXT: [[DATA0_ZEXT:%.*]] = zext i7 [[DATA0]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA0_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA0_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA1:%.*]] = extractvalue [4 x i7] [[DATA]], 1
; CHECK-NEXT: [[DATA1_ZEXT:%.*]] = zext i7 [[DATA1]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA1_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA1_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA2:%.*]] = extractvalue [4 x i7] [[DATA]], 2
; CHECK-NEXT: [[DATA2_ZEXT:%.*]] = zext i7 [[DATA2]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA2_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA2_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: [[DATA3:%.*]] = extractvalue [4 x i7] [[DATA]], 3
; CHECK-NEXT: [[DATA3_ZEXT:%.*]] = zext i7 [[DATA3]] to i8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA3_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA3_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2309,7 +2309,7 @@ define <vscale x 2 x i32> @load_nxv2i32(ptr addrspace(8) inreg %buf) {
define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_nxv2i32(
; CHECK-SAME: <vscale x 2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.nxv2i32(<vscale x 2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.nxv2i32(<vscale x 2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0, metadata [[META1]])
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2319,3 +2319,6 @@ define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf
!llvm.module.flags = !{!0}
!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 1}
+;.
+; CHECK: [[META1]] = !{}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
index ddccc4c4f59cd..167036eb7d725 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
@@ -88,51 +88,51 @@ define void @memcpy_known(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg %d
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -161,13 +161,13 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -181,13 +181,13 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 false)
@@ -202,7 +202,7 @@ define void @memcpy_known_byte(ptr addrspace(7) inreg %src, ptr addrspace(7) inr
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 1, i1 false)
@@ -217,19 +217,19 @@ define void @memcpy_known_tail(ptr addrspace(7) inreg %src, ptr addrspace(7) inr
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 8
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0)
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i32 [[SRC_OFF]], 14
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0)
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[DST_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 15, i1 false)
@@ -441,133 +441,133 @@ define void @memcpy_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(7) inre
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP3]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP3]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP3]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP3]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP3]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP3]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP3]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP3]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP3]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP3]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP3]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP3]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP3]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP3]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP3]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP3]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP3]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP3]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP3]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP3]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP3]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP3]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP3]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP3]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP3]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP3]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP3]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP3]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP3]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP3]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP3]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP3]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP3]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP3]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP3]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP3]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP3]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP3]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP3]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP3]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP3]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP3]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP3]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP3]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP3]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -596,13 +596,13 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -616,13 +616,13 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 true)
@@ -655,13 +655,13 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP6]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP6]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP6]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP7]] = add i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -674,7 +674,7 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[SRC_OFF]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0)
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[DST_OFF]], [[TMP9]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP13]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_LOOP_EXPANSION]]
@@ -697,52 +697,52 @@ define void @memcpy_known_p1_to_p7(ptr addrspace(1) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 16
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1034,51 +1034,51 @@ define void @memcpy.inline_known(ptr addrspace(7) inreg %src, ptr addrspace(7) i
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1107,13 +1107,13 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -1127,13 +1127,13 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 false)
@@ -1148,7 +1148,7 @@ define void @memcpy.inline_known_byte(ptr addrspace(7) inreg %src, ptr addrspace
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 1, i1 false)
@@ -1163,19 +1163,19 @@ define void @memcpy.inline_known_tail(ptr addrspace(7) inreg %src, ptr addrspace
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 8
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP5]], i32 0, i32 0)
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[TMP6]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i32 [[SRC_OFF]], 14
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP8]], i32 0, i32 0)
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[DST_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP9]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 15, i1 false)
@@ -1387,133 +1387,133 @@ define void @memcpy.inline_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP3]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP3]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP3]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP3]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP3]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP3]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP3]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP3]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP3]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP3]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP3]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP3]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP3]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP3]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP3]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP3]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP3]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP3]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP3]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP3]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP3]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP3]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP3]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP3]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP3]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP3]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP3]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP3]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP3]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP3]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP3]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP3]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP3]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP3]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP3]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP3]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP3]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP3]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP3]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP3]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP3]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP3]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP3]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP3]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP3]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1542,13 +1542,13 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
@@ -1562,13 +1562,13 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 true)
@@ -1601,13 +1601,13 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP6]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP6]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP6]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP7]] = add i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -1620,7 +1620,7 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[SRC_OFF]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP10]], i32 0, i32 0)
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[DST_OFF]], [[TMP9]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[TMP11]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP13]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_LOOP_EXPANSION]]
@@ -1643,52 +1643,52 @@ define void @memcpy.inline_known_p1_to_p7(ptr addrspace(1) inreg %src, ptr addrs
; CHECK-NEXT: [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 16
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1910,37 +1910,37 @@ define void @memset_known(ptr addrspace(7) inreg %ptr) {
; CHECK: [[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[TMP1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -1956,21 +1956,21 @@ define void @memset_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 false)
@@ -1982,7 +1982,7 @@ define void @memset_known_byte(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 1, i1 false)
@@ -1994,13 +1994,13 @@ define void @memset_known_tail(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[PTR_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 15, i1 false)
@@ -2017,133 +2017,133 @@ define void @memset_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTC]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP2]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP2]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP2]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP2]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP2]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP2]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP2]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP2]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP2]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP2]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP2]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP2]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP2]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP2]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP2]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP2]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP2]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP2]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP2]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP2]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP2]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP2]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP2]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP2]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP2]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP2]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP2]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP2]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP2]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP2]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP2]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP2]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP2]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP2]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP2]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP2]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP2]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP2]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP2]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP2]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP2]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP2]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP2]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP2]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP2]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -2159,21 +2159,21 @@ define void @memset_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 true)
@@ -2192,13 +2192,13 @@ define void @memset_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %length) {
; CHECK: [[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[TMP2]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i32 [[TMP2]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -2209,7 +2209,7 @@ define void @memset_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %length) {
; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]] ], [ [[TMP10:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP4]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[PTR_OFF]], [[TMP8]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMSET_POST_LOOP_EXPANSION]]
@@ -2234,37 +2234,37 @@ define void @memset.inline_known(ptr addrspace(7) inreg %ptr) {
; CHECK: [[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[TMP1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -2280,21 +2280,21 @@ define void @memset.inline_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 false)
@@ -2306,7 +2306,7 @@ define void @memset.inline_known_byte(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 1, i1 false)
@@ -2318,13 +2318,13 @@ define void @memset.inline_known_tail(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 bitcast (<8 x i8> splat (i8 1) to i64), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 bitcast (<4 x i8> splat (i8 1) to i32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 bitcast (<2 x i8> splat (i8 1) to i16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[PTR_OFF]], 14
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 15, i1 false)
@@ -2341,133 +2341,133 @@ define void @memset.inline_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTC]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP2]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP2]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP2]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP2]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP2]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP2]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP2]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP2]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP2]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP2]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP2]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP2]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP2]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP2]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP2]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP2]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP2]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP2]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP2]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP2]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP2]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP2]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP2]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP2]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP2]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP2]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP2]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP2]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP2]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP2]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP2]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP2]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP2]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP2]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP2]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP2]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP2]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP2]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP2]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP2]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP2]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP2]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP2]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP2]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP2]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP2]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP2]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP2]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -2483,21 +2483,21 @@ define void @memset.inline_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 true)
@@ -2516,13 +2516,13 @@ define void @memset.inline_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %lengt
; CHECK: [[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[TMP2]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i32 [[TMP2]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -2533,7 +2533,7 @@ define void @memset.inline_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %lengt
; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]] ], [ [[TMP10:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP4]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[PTR_OFF]], [[TMP8]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMSET_POST_LOOP_EXPANSION]]
@@ -2559,37 +2559,37 @@ define void @memset_pattern_known(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[LOADSTORELOOP]] ]
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP5]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP5]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP5]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP5]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP5]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP5]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP5]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP5]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP5]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP5]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP5]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP5]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP5]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP5]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP5]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 64
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP3]], label %[[LOADSTORELOOP]], label %[[SPLIT:.*]]
@@ -2610,7 +2610,7 @@ define void @memset_pattern_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[DOTIDX:%.*]] = mul nsw i32 [[TMP1]], 4
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 32
; CHECK-NEXT: br i1 [[TMP4]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]], label %[[SPLIT:.*]]
@@ -2632,133 +2632,133 @@ define void @memset_pattern_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[DOTC]], 256
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP8]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP8]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP8]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP8]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP8]], 20
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP8]], 24
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP8]], 28
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP8]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP8]], 36
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP8]], 40
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP8]], 44
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP8]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP8]], 52
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP8]], 56
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP8]], 60
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP8]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP8]], 68
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP8]], 72
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP8]], 76
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP8]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP8]], 84
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP8]], 88
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP8]], 92
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP8]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP8]], 100
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP8]], 104
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP8]], 108
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP8]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP8]], 116
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP8]], 120
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP8]], 124
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP8]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP8]], 132
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP8]], 136
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP8]], 140
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP8]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP8]], 148
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP8]], 152
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP8]], 156
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP8]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP8]], 164
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP8]], 168
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP8]], 172
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP8]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP8]], 180
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP8]], 184
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP8]], 188
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP8]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP8]], 196
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP8]], 200
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP8]], 204
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP8]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP8]], 212
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP8]], 216
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP8]], 220
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP8]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP8]], 228
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP8]], 232
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP8]], 236
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP8]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP8]], 244
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP8]], 248
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP8]], 252
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i64 [[TMP1]], 64
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP5]], 8192
; CHECK-NEXT: br i1 [[TMP3]], label %[[LOADSTORELOOP]], label %[[SPLIT:.*]]
@@ -2779,7 +2779,7 @@ define void @memset_pattern_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]] ]
; CHECK-NEXT: [[DOTIDX:%.*]] = mul nsw i32 [[TMP1]], 4
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648, metadata [[META0]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 32
; CHECK-NEXT: br i1 [[TMP4]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]], label %[[SPLIT:.*]]
@@ -2803,13 +2803,13 @@ define void @memset_pattern_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %leng
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[LOADSTORELOOP]] ]
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP4]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP4]], 4
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP4]], 8
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP4]], 12
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP5]] = add i32 [[LOOP_INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[LOADSTORELOOP]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_COND]]
@@ -2821,7 +2821,7 @@ define void @memset_pattern_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %leng
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP8]], [[RESIDUAL_LOOP_INDEX]]
; CHECK-NEXT: [[DOTIDX:%.*]] = mul nsw i32 [[TMP2]], 4
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[TMP10]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP1]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_BODY]], label %[[SPLIT]]
@@ -2849,3 +2849,6 @@ define void @llvm_amdgcn_load_to_lds(ptr addrspace(7) inreg %p, ptr addrspace(3)
call void @llvm.amdgcn.load.to.lds.p7(ptr addrspace(7) %q, ptr addrspace(3) %l, i32 4, i32 16, i32 0)
ret void
}
+;.
+; CHECK: [[META0]] = !{}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
index 74735529d723d..de955eebd1518 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll
@@ -16,10 +16,10 @@ define void @loads(ptr addrspace(8) %buf) {
; CHECK-NEXT: [[VOLATILE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
; CHECK-NEXT: [[VOLATILE_NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648), !nontemporal [[META0]]
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2:![0-9]+]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
-; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
+; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.atomic.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3:![0-9]+]])
; CHECK-NEXT: fence acquire
; CHECK-NEXT: ret void
;
@@ -47,17 +47,17 @@ define void @loads(ptr addrspace(8) %buf) {
define void @stores(ptr addrspace(8) %buf, float %f, <4 x float> %f4) {
; CHECK-LABEL: define void @stores
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], <4 x float> [[F4:%.*]]) {
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !nontemporal [[META0]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648), !nontemporal [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META1]]), !nontemporal [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META1]])
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META1]]), !nontemporal [[META0]]
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence release
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META3]])
; CHECK-NEXT: ret void
;
%base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -82,46 +82,46 @@ define void @atomicrmw(ptr addrspace(8) %buf, float %f, i32 %i) {
; CHECK-LABEL: define void @atomicrmw
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], i32 [[I:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[XCHG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.swap.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[XCHG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.swap.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[ADD:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[ADD:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[SUB:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[SUB:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[AND:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.and.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[AND:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.and.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[OR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.or.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[OR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.or.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[XOR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.xor.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[XOR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.xor.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[MIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[MIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[MAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[MAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[FADD:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[FADD:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[FMAX:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[FMAX:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[FMIN:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[FMIN:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: ret void
;
@@ -154,7 +154,7 @@ define {i32, i1} @cmpxchg(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
; CHECK-LABEL: define { i32, i1 } @cmpxchg
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648)
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]]
@@ -172,7 +172,7 @@ define {i32, i1} @cmpxchg_weak(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
; CHECK-LABEL: define { i32, i1 } @cmpxchg_weak
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) {
; CHECK-NEXT: fence syncscope("wavefront") release
-; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0, metadata [[META2]])
; CHECK-NEXT: fence syncscope("wavefront") acquire
; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
index 21d0caa40ea3e..76fe2f662b393 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -56,10 +56,10 @@ define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace
; CHECK-NEXT: #dbg_value(ptr addrspace(8) [[AUX_PTR_2_PTR_RSRC]], [[META21]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG34]])
; CHECK-NEXT: [[BUF_PTR_4_LEGAL:%.*]] = bitcast i160 [[BUF_PTR_4]] to <5 x i32>, !dbg [[DBG35:![0-9]+]]
; CHECK-NEXT: [[BUF_PTR_4_SLICE_0:%.*]] = shufflevector <5 x i32> [[BUF_PTR_4_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, !dbg [[DBG35]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0), !dbg [[DBG35]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0, metadata [[META8:![0-9]+]]), !dbg [[DBG35]]
; CHECK-NEXT: [[AUX_PTR_2_PTR_PART_4:%.*]] = add nuw i32 [[AUX_PTR_2_PTR_OFF]], 16, !dbg [[DBG35]]
; CHECK-NEXT: [[BUF_PTR_4_SLICE_4:%.*]] = extractelement <5 x i32> [[BUF_PTR_4_LEGAL]], i64 4, !dbg [[DBG35]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0), !dbg [[DBG35]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0, metadata [[META8]]), !dbg [[DBG35]]
; CHECK-NEXT: ret float [[RET]], !dbg [[DBG36:![0-9]+]]
;
%buf.ptr.var = alloca ptr addrspace(7), align 32, addrspace(5), !dbg !20
diff --git a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
index 0113437240236..f71844626cc91 100644
--- a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
+++ b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
; FNATTRS-NEXT: [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[P]], i16 0, i64 4, i32 822243328)
; FNATTRS-NEXT: [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[Q]], i16 0, i64 4, i32 822243328)
; FNATTRS-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[P_RSRC]], i32 0, i32 0, i32 0)
-; FNATTRS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) [[Q_RSRC]], i32 0, i32 0, i32 0)
+; FNATTRS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]])
; FNATTRS-NEXT: ret void
;
; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
@@ -18,7 +18,7 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
; ATTRIBUTOR-NEXT: [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[P]], i16 0, i64 4, i32 822243328) #[[ATTR4:[0-9]+]]
; ATTRIBUTOR-NEXT: [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr [[Q]], i16 0, i64 4, i32 822243328) #[[ATTR4]]
; ATTRIBUTOR-NEXT: [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) readonly captures(none) [[P_RSRC]], i32 0, i32 0, i32 0) #[[ATTR5:[0-9]+]]
-; ATTRIBUTOR-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0) #[[ATTR6:[0-9]+]]
+; ATTRIBUTOR-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0, metadata [[META0:![0-9]+]]) #[[ATTR6:[0-9]+]]
; ATTRIBUTOR-NEXT: ret void
;
%p.rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 0, i64 4, i32 822243328)
diff --git a/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll b/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
index 51e2fb003d41b..65e8d85ca374a 100644
--- a/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
+++ b/llvm/test/Transforms/LICM/AMDGPU/buffer-rsrc-ptrs.ll
@@ -13,7 +13,7 @@ define void @hoistable_noalias(ptr addrspace(8) noalias %p, ptr addrspace(8) noa
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0:![0-9]+]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -47,7 +47,7 @@ define void @not_hoistable_may_alias(ptr addrspace(8) %p, ptr addrspace(8) %q, i
; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[NOT_HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -75,13 +75,13 @@ define void @hoistable_alias_scope(ptr addrspace(8) %p, ptr addrspace(8) %q, i32
; CHECK-LABEL: define void @hoistable_alias_scope
; CHECK-SAME: (ptr addrspace(8) [[P:%.*]], ptr addrspace(8) [[Q:%.*]], i32 [[BOUND:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META1:![0-9]+]], !noalias [[META4:![0-9]+]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -129,7 +129,7 @@ define void @not_hoistable_buffer_construction(ptr addrspace(1) noalias %p.globa
; CHECK-NEXT: [[NOT_HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0)
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[NOT_HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -173,7 +173,7 @@ define void @hoistable_buffer_construction_intrinsic(ptr addrspace(1) noalias %p
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]])
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
@@ -212,13 +212,13 @@ define void @hoistable_buffer_construction_alias_scope(ptr addrspace(1) %p.globa
; CHECK-NEXT: [[Q_EXT:%.*]] = zext i48 [[Q_TRUNC]] to i128
; CHECK-NEXT: [[P:%.*]] = inttoptr i128 [[P_EXT]] to ptr addrspace(8)
; CHECK-NEXT: [[Q:%.*]] = inttoptr i128 [[Q_EXT]] to ptr addrspace(8)
-; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT: [[HOISTABLE:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) [[Q]], i32 0, i32 0, i32 0, i32 0), !alias.scope [[META1]], !noalias [[META4]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[ORIG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[INC:%.*]] = add i32 [[HOISTABLE]], [[ORIG]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[INC]], ptr addrspace(8) [[P]], i32 [[I]], i32 0, i32 0, metadata [[META0]]), !alias.scope [[META4]], !noalias [[META1]]
; CHECK-NEXT: [[NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[NEXT]], [[BOUND]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[TAIL:%.*]]
>From 8ee783c40a6dce6f181b31333c2659d88837f4e0 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 6 Aug 2026 11:21:47 +0200
Subject: [PATCH 4/4] clang
---
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 24 ++++++++++---
.../CodeGenOpenCL/builtins-amdgcn-gfx13.cl | 8 ++---
.../builtins-amdgcn-raw-buffer-atomic-add.cl | 9 +++--
.../builtins-amdgcn-raw-buffer-atomic-max.cl | 7 ++--
.../builtins-amdgcn-raw-buffer-store.cl | 36 +++++++++----------
5 files changed, 52 insertions(+), 32 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 29199f1726c1a..fa90a335d8cc3 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -31,6 +31,20 @@ using namespace llvm;
namespace {
+// Emits a raw.ptr.buffer.{store,atomic.*} intrinsic call with an appended
+// empty syncscope metadata arg, since the builtin form carries no scope info.
+static Value *emitAMDGPURawPtrBufferBuiltinWithScope(CodeGenFunction &CGF,
+ const CallExpr *E,
+ unsigned IntrinsicID) {
+ llvm::SmallVector<Value *, 6> Args;
+ for (unsigned I = 0; I < 5; ++I)
+ Args.push_back(CGF.EmitScalarExpr(E->getArg(I)));
+ Args.push_back(MetadataAsValue::get(CGF.getLLVMContext(),
+ MDNode::get(CGF.getLLVMContext(), {})));
+ Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Args[0]->getType());
+ return CGF.Builder.CreateCall(F, Args);
+}
+
static Value *emitAMDGPUSBufferLoadBuiltin(CodeGenFunction &CGF,
const CallExpr *E) {
llvm::Type *RetTy = CGF.ConvertType(E->getType());
@@ -2090,7 +2104,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_store);
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f32:
case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f16:
@@ -2155,19 +2169,19 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
EmitScalarExpr(E->getArg(4))});
}
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_add_i32:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_add);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f64:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin);
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32:
case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64:
- return emitBuiltinWithOneOverloadedType<5>(
+ return emitAMDGPURawPtrBufferBuiltinWithScope(
*this, E, Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax);
case AMDGPU::BI__builtin_amdgcn_s_buffer_load_i32:
case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2i32:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl
index f5d341b31991c..f9d7a3d3b16ac 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx13.cl
@@ -68,28 +68,28 @@ void test_cvt_scalef32_pk32_fp6_f32(global uint6 *out, float32 srcf32, float src
// CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[TMP0]], ptr addrspace(8) [[TMP1]], i32 [[TMP2]], i32 [[TMP3]], i32 0)
+// CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[TMP0]], ptr addrspace(8) [[TMP1]], i32 [[TMP2]], i32 [[TMP3]], i32 0, metadata [[META3:![0-9]+]])
// CHECK-NEXT: [[TMP5:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[FOUT_ADDR]], align 8
// CHECK-NEXT: store float [[TMP4]], ptr addrspace(1) [[TMP5]], align 4
// CHECK-NEXT: [[TMP6:%.*]] = load float, ptr addrspace(5) [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP7:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[TMP6]], ptr addrspace(8) [[TMP7]], i32 [[TMP8]], i32 [[TMP9]], i32 0)
+// CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[TMP6]], ptr addrspace(8) [[TMP7]], i32 [[TMP8]], i32 [[TMP9]], i32 0, metadata [[META3]])
// CHECK-NEXT: [[TMP11:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[FOUT_ADDR]], align 8
// CHECK-NEXT: store float [[TMP10]], ptr addrspace(1) [[TMP11]], align 4
// CHECK-NEXT: [[TMP12:%.*]] = load double, ptr addrspace(5) [[D_ADDR]], align 8
// CHECK-NEXT: [[TMP13:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double [[TMP12]], ptr addrspace(8) [[TMP13]], i32 [[TMP14]], i32 [[TMP15]], i32 0)
+// CHECK-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double [[TMP12]], ptr addrspace(8) [[TMP13]], i32 [[TMP14]], i32 [[TMP15]], i32 0, metadata [[META3]])
// CHECK-NEXT: [[TMP17:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[DOUT_ADDR]], align 8
// CHECK-NEXT: store double [[TMP16]], ptr addrspace(1) [[TMP17]], align 8
// CHECK-NEXT: [[TMP18:%.*]] = load double, ptr addrspace(5) [[D_ADDR]], align 8
// CHECK-NEXT: [[TMP19:%.*]] = load ptr addrspace(8), ptr addrspace(5) [[RSRC_ADDR]], align 16
// CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(5) [[OFFSET_ADDR]], align 4
// CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(5) [[SOFFSET_ADDR]], align 4
-// CHECK-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[TMP18]], ptr addrspace(8) [[TMP19]], i32 [[TMP20]], i32 [[TMP21]], i32 0)
+// CHECK-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[TMP18]], ptr addrspace(8) [[TMP19]], i32 [[TMP20]], i32 [[TMP21]], i32 0, metadata [[META3]])
// CHECK-NEXT: [[TMP23:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[DOUT_ADDR]], align 8
// CHECK-NEXT: store double [[TMP22]], ptr addrspace(1) [[TMP23]], align 8
// CHECK-NEXT: ret void
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl
index 95abeb92e59e7..079d36ccfef40 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-add.cl
@@ -10,7 +10,7 @@ typedef half __attribute__((ext_vector_type(2))) float16x2_t;
// CHECK-LABEL: define dso_local i32 @test_atomic_add_i32(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], i32 noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret i32 [[TMP0]]
//
int test_atomic_add_i32(__amdgpu_buffer_rsrc_t rsrc, int x, int offset, int soffset) {
@@ -20,7 +20,7 @@ int test_atomic_add_i32(__amdgpu_buffer_rsrc_t rsrc, int x, int offset, int soff
// CHECK-LABEL: define dso_local float @test_atomic_fadd_f32(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], float noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret float [[TMP0]]
//
float test_atomic_fadd_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int soffset) {
@@ -30,9 +30,12 @@ float test_atomic_fadd_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int
// CHECK-LABEL: define dso_local <2 x half> @test_atomic_fadd_v2f16(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], <2 x half> noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret <2 x half> [[TMP0]]
//
float16x2_t test_atomic_fadd_v2f16(__amdgpu_buffer_rsrc_t rsrc, float16x2_t x, int offset, int soffset) {
return __builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16(x, rsrc, offset, soffset, 0);
}
+//.
+// CHECK: [[META8]] = !{}
+//.
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl
index feb8682d522b1..21d2c627d5a7b 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-atomic-max.cl
@@ -8,7 +8,7 @@
// CHECK-LABEL: define dso_local float @test_atomic_fmax_f32(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], float noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret float [[TMP0]]
//
float test_atomic_fmax_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int soffset) {
@@ -18,9 +18,12 @@ float test_atomic_fmax_f32(__amdgpu_buffer_rsrc_t rsrc, float x, int offset, int
// CHECK-LABEL: define dso_local double @test_atomic_fmax_f64(
// CHECK-SAME: ptr addrspace(8) nofree captures(none) [[RSRC:%.*]], double noundef [[X:%.*]], i32 noundef [[OFFSET:%.*]], i32 noundef [[SOFFSET:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = tail call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0)
+// CHECK-NEXT: [[TMP0:%.*]] = tail call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double [[X]], ptr addrspace(8) [[RSRC]], i32 [[OFFSET]], i32 [[SOFFSET]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret double [[TMP0]]
//
double test_atomic_fmax_f64(__amdgpu_buffer_rsrc_t rsrc, double x, int offset, int soffset) {
return __builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64(x, rsrc, offset, soffset, 0);
}
+//.
+// CHECK: [[META8]] = !{}
+//.
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
index e2dfa50ee3a2d..c07126a1584b4 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-raw-buffer-store.cl
@@ -11,7 +11,7 @@ typedef unsigned int v4u32 __attribute__((ext_vector_type(4)));
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b8(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -20,7 +20,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8(u8 vdata, __amdgpu_buffer_rsrc_t rsrc,
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b16(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b16(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -29,7 +29,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16(u16 vdata, __amdgpu_buffer_rsrc_t rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b32(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b32(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -38,7 +38,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32(u32 vdata, __amdgpu_buffer_rsrc_t rsrc
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b64(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b64(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -47,7 +47,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64(v2u32 vdata, __amdgpu_buffer_rsrc_t rs
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b96(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b96(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -56,7 +56,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96(v3u32 vdata, __amdgpu_buffer_rsrc_t rs
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b128(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b128(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -65,7 +65,7 @@ void test_amdgcn_raw_ptr_buffer_store_b128(v4u32 vdata, __amdgpu_buffer_rsrc_t r
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -74,7 +74,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8_non_const_offset(u8 vdata, __amdgpu_buf
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -83,7 +83,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16_non_const_offset(u16 vdata, __amdgpu_b
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -92,7 +92,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32_non_const_offset(u32 vdata, __amdgpu_b
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -101,7 +101,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64_non_const_offset(v2u32 vdata, __amdgpu
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -110,7 +110,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96_non_const_offset(v3u32 vdata, __amdgpu
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 [[OFFSET:%.*]], i32 0, i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -119,7 +119,7 @@ void test_amdgcn_raw_ptr_buffer_store_b128_non_const_offset(v4u32 vdata, __amdgp
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(u8 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -128,7 +128,7 @@ void test_amdgcn_raw_ptr_buffer_store_b8_non_const_soffset(u8 vdata, __amdgpu_bu
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(u16 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -137,7 +137,7 @@ void test_amdgcn_raw_ptr_buffer_store_b16_non_const_soffset(u16 vdata, __amdgpu_
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -146,7 +146,7 @@ void test_amdgcn_raw_ptr_buffer_store_b32_non_const_soffset(u32 vdata, __amdgpu_
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(v2u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -155,7 +155,7 @@ void test_amdgcn_raw_ptr_buffer_store_b64_non_const_soffset(v2u32 vdata, __amdgp
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(v3u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
@@ -164,7 +164,7 @@ void test_amdgcn_raw_ptr_buffer_store_b96_non_const_soffset(v3u32 vdata, __amdgp
// CHECK-LABEL: @test_amdgcn_raw_ptr_buffer_store_b128_non_const_soffset(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0)
+// CHECK-NEXT: tail call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[VDATA:%.*]], ptr addrspace(8) [[RSRC:%.*]], i32 0, i32 [[SOFFSET:%.*]], i32 0, metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_raw_ptr_buffer_store_b128_non_const_soffset(v4u32 vdata, __amdgpu_buffer_rsrc_t rsrc, int offset, int soffset) {
More information about the llvm-commits
mailing list